The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Revert "Async workspace provisioning" (#78172)
Alexander Polyankin7413348d6899
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex 87a9bd52e22..216b43e77e2 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -209,8 +209,6 @@metabase-enterprise.workspaces.test-util/create-ready-ws!metabase-enterprise.workspaces.core/clear-instance-workspace!metabase-enterprise.workspaces.core/set-instance-workspace!- metabase-enterprise.workspaces.instance/clear-instance-workspace!- metabase-enterprise.workspaces.instance/set-instance-workspace!metabase.workspaces.core/clear-instance-workspace!metabase.workspaces.core/set-instance-workspace!;; Safe in tests, where we typically use a private, temporary index per thread.diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 74b26422b99..1c93c104dfa 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2870,8 +2870,8 @@enterprise/advanced-config{:team "UX West"- :api #{metabase-enterprise.advanced-config.api.logs- metabase-enterprise.advanced-config.api.routes+ :api #{metabase-enterprise.advanced-config.api+ metabase-enterprise.advanced-config.api.logsmetabase-enterprise.advanced-config.filemetabase-enterprise.advanced-config.initmetabase-enterprise.advanced-config.settings}@@ -2888,7 +2888,6 @@syncusersutil- enterprise/remote-syncenterprise/workspaces}:model-imports #{:model/ApiKey :model/Database :model/User}}@@ -3015,20 +3014,6 @@:model/QueryCache:model/QueryExecution}}- enterprise/checker- {:team "Graphy"- ;; todo: add ability to prevent usage (transitive even) of app db- :api #{}- :uses #{classloader- lib- models- plugins- query-processor- sql-tools- util- enterprise/dependencies}- :model-imports #{:model/Database}}-enterprise/cloud-add-ons{:team "Cloud":api #{metabase-enterprise.cloud-add-ons.api}@@ -3184,8 +3169,7 @@enterprise/dependencies{:team "Graphy"- :api #{metabase-enterprise.dependencies.analysis- metabase-enterprise.dependencies.api+ :api #{metabase-enterprise.dependencies.apimetabase-enterprise.dependencies.coremetabase-enterprise.dependencies.initmetabase-enterprise.dependencies.models.dependency}@@ -3620,7 +3604,6 @@:api #{metabase-enterprise.serialization.apimetabase-enterprise.serialization.cmdmetabase-enterprise.serialization.core- metabase-enterprise.serialization.initmetabase-enterprise.serialization.v2.backfill-ids}:friends #{enterprise/memoize-monitor}:uses #{analytics@@ -3637,9 +3620,7 @@searchserversetup- sync- util- warehouse-schema}+ util}:model-imports :bypass}enterprise/snippet-collections@@ -3803,8 +3784,7 @@settingssql-toolsutil- workspaces- enterprise/remote-sync}+ workspaces}:model-exports #{:model/Workspace}:model-imports #{:model/Database:model/Tablediff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex 8555c69f1ea..0ad9849f5cb 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -6,14 +6,14 @@;; :all is the vector-less ignore form, which suppresses every linter on the next form.{:ignore-counts {:aliased-namespace-symbol 3:all 50- :case-symbol-test 2+ :case-symbol-test 1:clojure-lsp/unused-public-var 9:def-fn 1:deprecated-namespace 97:deprecated-var 81:discouraged-java-method 3:discouraged-namespace 81- :discouraged-var 108+ :discouraged-var 106:equals-true 1:main-without-gen-class 1:metabase/defsetting-namespace 7@@ -27,7 +27,7 @@:metabase/validate-defendpoint-has-response-schema 444:metabase/validate-defendpoint-query-params-use-kebab-case 40:metabase/validate-defendpoint-route-uses-kebab-case 44- :metabase/validate-deftest 10+ :metabase/validate-deftest 8:missing-docstring 10:missing-protocol-method 3:non-arg-vec-return-type-hint 1diff --git a/e2e/support/helpers/api/index.ts b/e2e/support/helpers/api/index.tsindex 522efdcb689..9a018c11418 100644--- a/e2e/support/helpers/api/index.ts+++ b/e2e/support/helpers/api/index.ts@@ -54,3 +54,4 @@ export {updateSetting,} from "./updateSetting";export { updateSnippet } from "./updateSnippet";+export { clearWorkspaceInstanceConfig } from "./setWorkspaceInstanceConfig";diff --git a/e2e/support/helpers/api/setWorkspaceInstanceConfig.ts b/e2e/support/helpers/api/setWorkspaceInstanceConfig.tsnew file mode 100644index 00000000000..4c9a66c5610--- /dev/null+++ b/e2e/support/helpers/api/setWorkspaceInstanceConfig.ts@@ -0,0 +1,3 @@+export function clearWorkspaceInstanceConfig() {+ return cy.request("DELETE", "/api/ee/workspace-instance/current");+}diff --git a/e2e/support/helpers/e2e-workspace-helpers.ts b/e2e/support/helpers/e2e-workspace-helpers.tsnew file mode 100644index 00000000000..cdc2e5ecc52--- /dev/null+++ b/e2e/support/helpers/e2e-workspace-helpers.ts@@ -0,0 +1,105 @@+import yaml from "js-yaml";++import type { AdvancedConfig } from "metabase-types/api";++import { modal } from "./e2e-ui-elements-helpers";++export const WorkspaceListPage = {+ get: () => cy.findByTestId("workspace-list-page"),+ visit: () => {+ cy.visit("/data-studio/workspaces");+ WorkspaceListPage.get().should("be.visible");+ },+ newButton: ({ primary = true }: { primary?: boolean } = {}) =>+ WorkspaceListPage.get().findByRole("button", {+ name: primary ? "Create a workspace" : "New",+ }),+ setupInstanceButton: () =>+ WorkspaceListPage.get().findByRole("button", {+ name: "Upload a workspace config",+ }),+ workspaceList: () => WorkspaceListPage.get().findByTestId("workspace-list"),+ workspace: (name: string) =>+ WorkspaceListPage.get().findByRole("region", { name }),+ workspaceMenuButton: (name: string) =>+ WorkspaceListPage.workspace(name).findByRole("button", {+ name: "Workspace options",+ }),+ renameMenuItem: () => cy.findByRole("menuitem", { name: "Rename" }),+ downloadConfigMenuItem: () =>+ cy.findByRole("menuitem", { name: /Download config\.yml/ }),+ deleteMenuItem: () => cy.findByRole("menuitem", { name: "Delete" }),+};++export const NewWorkspaceModal = {+ get: () => modal(),+ nameInput: () => NewWorkspaceModal.get().findByLabelText("Name"),+ databaseCheckbox: (name: string) =>+ NewWorkspaceModal.get().findByRole("checkbox", { name }),+ createButton: () =>+ NewWorkspaceModal.get().findByRole("button", { name: "Create workspace" }),+ cancelButton: () =>+ NewWorkspaceModal.get().findByRole("button", { name: "Cancel" }),+};++export const RenameWorkspaceModal = {+ get: () => modal(),+ nameInput: () => RenameWorkspaceModal.get().findByLabelText("Name"),+ renameButton: () =>+ RenameWorkspaceModal.get().findByRole("button", { name: "Rename" }),+ cancelButton: () =>+ RenameWorkspaceModal.get().findByRole("button", { name: "Cancel" }),+};++export const DeleteWorkspaceModal = {+ get: () => modal(),+ confirmButton: () =>+ DeleteWorkspaceModal.get().findByRole("button", {+ name: "Delete workspace",+ }),+ cancelButton: () =>+ DeleteWorkspaceModal.get().findByRole("button", { name: "Cancel" }),+};++export const SetupWorkspaceModal = {+ get: () => modal(),+ configInput: () => SetupWorkspaceModal.get().get('input[type="file"]'),+ setupButton: () =>+ SetupWorkspaceModal.get().findByRole("button", { name: "Set up" }),+ cancelButton: () =>+ SetupWorkspaceModal.get().findByRole("button", { name: "Cancel" }),+ uploadConfig: (config: AdvancedConfig) => {+ SetupWorkspaceModal.configInput().selectFile(+ {+ contents: Cypress.Buffer.from(yaml.dump(config)),+ fileName: "config.yml",+ mimeType: "application/yaml",+ },+ { force: true },+ );+ },+};++export const CurrentWorkspacePage = {+ get: () => cy.findByTestId("current-workspace-page"),+ visit: () => {+ cy.visit("/data-studio/workspaces");+ CurrentWorkspacePage.get().should("be.visible");+ },+ database: (name: string) =>+ CurrentWorkspacePage.get().findByRole("region", { name }),+ remappingRow: (canonicalName: string) =>+ CurrentWorkspacePage.get().findByText(canonicalName),+ leaveButton: () =>+ CurrentWorkspacePage.get().findByRole("button", {+ name: "Leave workspace",+ }),+};++export const LeaveWorkspaceModal = {+ get: () => modal(),+ confirmButton: () =>+ LeaveWorkspaceModal.get().findByRole("button", { name: "Leave workspace" }),+ cancelButton: () =>+ LeaveWorkspaceModal.get().findByRole("button", { name: "Cancel" }),+};diff --git a/e2e/support/helpers/index.ts b/e2e/support/helpers/index.tsindex bf82f36ce08..1d4ffd84e7a 100644--- a/e2e/support/helpers/index.ts+++ b/e2e/support/helpers/index.ts@@ -66,4 +66,5 @@ export * from "./e2e-upload-helpers";export * from "./e2e-users-helpers";export * from "./e2e-visual-tests-helpers";export * from "./e2e-viz-settings-helpers";+export * from "./e2e-workspace-helpers";export * from "./embedding-sdk-helpers/constants";diff --git a/e2e/support/timings.json b/e2e/support/timings.jsonindex 5ea5a6c248a..20a8b30a5d8 100644--- a/e2e/support/timings.json+++ b/e2e/support/timings.json@@ -1659,6 +1659,14 @@{"spec": "../test/scenarios/visualizations-tabular/visualizations-tabular-reproductions.cy.spec.js","duration": 113946+ },+ {+ "spec": "../test/scenarios/workspaces/workspace-instance.cy.spec.ts",+ "duration": 42522+ },+ {+ "spec": "../test/scenarios/workspaces/workspace-manager.cy.spec.ts",+ "duration": 9359}]-}+}\ No newline at end of filediff --git a/e2e/test/scenarios/workspaces/workspace-instance.cy.spec.ts b/e2e/test/scenarios/workspaces/workspace-instance.cy.spec.tsnew file mode 100644index 00000000000..42fb7f9845a--- /dev/null+++ b/e2e/test/scenarios/workspaces/workspace-instance.cy.spec.ts@@ -0,0 +1,286 @@+import {+ QA_DB_CREDENTIALS,+ QA_MYSQL_PORT,+ QA_POSTGRES_PORT,+ WRITABLE_DB_ID,+} from "e2e/support/cypress_data";+import type { AdvancedConfig } from "metabase-types/api";++const { H } = cy;++const WORKSPACE_NAME = "E2E workspace instance";+const ROW_COUNT = 3;++const POSTGRES_DB_NAME = "Writable Postgres12";+const POSTGRES_INPUT_SCHEMA = "Domestic";+const POSTGRES_OUTPUT_SCHEMA = "mb__isolation";+const POSTGRES_SOURCE_TABLE = "Animals";+const POSTGRES_TARGET_TABLE = "transform_table";+const POSTGRES_TARGET_TABLE_DISPLAY_NAME = "Transform Table";++const POSTGRES_CONFIG: AdvancedConfig = {+ version: 1,+ config: {+ databases: [+ {+ name: POSTGRES_DB_NAME,+ engine: "postgres",+ details: {+ host: QA_DB_CREDENTIALS.host,+ port: QA_POSTGRES_PORT,+ dbname: "writable_db",+ user: QA_DB_CREDENTIALS.user,+ password: QA_DB_CREDENTIALS.password,+ ssl: false,+ },+ },+ ],+ workspace: {+ name: WORKSPACE_NAME,+ databases: {+ [POSTGRES_DB_NAME]: {+ input_schemas: [POSTGRES_INPUT_SCHEMA],+ output: { schema: POSTGRES_OUTPUT_SCHEMA },+ },+ },+ },+ },+};++const MYSQL_DB_NAME = "Writable MySQL8";+const MYSQL_OUTPUT_DATABASE = "mb__isolation";+const MYSQL_SOURCE_TABLE = "scoreboard_actions";+const MYSQL_TARGET_TABLE = "transform_table";+const MYSQL_TARGET_TABLE_DISPLAY_NAME = "Transform Table";++const MYSQL_CONFIG: AdvancedConfig = {+ version: 1,+ config: {+ databases: [+ {+ name: MYSQL_DB_NAME,+ engine: "mysql",+ details: {+ host: QA_DB_CREDENTIALS.host,+ port: QA_MYSQL_PORT,+ dbname: "writable_db",+ user: "root",+ password: QA_DB_CREDENTIALS.password,+ ssl: false,+ },+ },+ ],+ workspace: {+ name: WORKSPACE_NAME,+ databases: {+ [MYSQL_DB_NAME]: {+ input_schemas: [],+ output: { db: MYSQL_OUTPUT_DATABASE },+ },+ },+ },+ },+};++describe("scenarios > workspaces > workspace instance", () => {+ describe("postgres", () => {+ beforeEach(() => {+ H.restore("postgres-writable");+ cy.signInAsAdmin();+ H.activateToken("bleeding-edge");+ H.resetTestTable({ type: "postgres", table: "multi_schema" });+ H.queryWritableDB(+ `CREATE SCHEMA IF NOT EXISTS ${POSTGRES_OUTPUT_SCHEMA}`,+ "postgres",+ );+ H.resyncDatabase({ dbId: WRITABLE_DB_ID });+ });++ afterEach(() => {+ H.clearWorkspaceInstanceConfig();+ H.queryWritableDB(+ `DROP SCHEMA IF EXISTS ${POSTGRES_OUTPUT_SCHEMA} CASCADE`,+ "postgres",+ );+ });++ it("runs a transform, surfaces the remapping on the instance page, and queries the remapped table", () => {+ cy.log("set up the workspace");+ H.WorkspaceListPage.visit();+ H.WorkspaceListPage.setupInstanceButton().click();+ H.SetupWorkspaceModal.uploadConfig(POSTGRES_CONFIG);+ H.SetupWorkspaceModal.setupButton().click();+ H.CurrentWorkspacePage.get().should("be.visible");++ cy.log("create and run a transform via the API");+ createAndRunTransform({+ sourceTable: POSTGRES_SOURCE_TABLE,+ sourceSchema: POSTGRES_INPUT_SCHEMA,+ targetTable: POSTGRES_TARGET_TABLE,+ targetSchema: POSTGRES_INPUT_SCHEMA,+ });++ cy.log("instance page shows the remapping for the transform target");+ H.CurrentWorkspacePage.visit();+ H.CurrentWorkspacePage.database(POSTGRES_DB_NAME)+ .should(+ "contain.text",+ `${POSTGRES_INPUT_SCHEMA}/${POSTGRES_TARGET_TABLE}`,+ )+ .and(+ "contain.text",+ `${POSTGRES_OUTPUT_SCHEMA}/${POSTGRES_INPUT_SCHEMA}__${POSTGRES_TARGET_TABLE}`,+ );++ cy.log("native query is rewritten to workspace table");+ H.startNewNativeQuestion({+ database: WRITABLE_DB_ID,+ query: `SELECT * FROM "${POSTGRES_INPUT_SCHEMA}"."${POSTGRES_TARGET_TABLE}"`,+ });+ H.runNativeQuery();+ H.assertQueryBuilderRowCount(ROW_COUNT);++ cy.log("mbql query is rewritten to workspace table");+ H.startNewQuestion();+ H.miniPicker().findByText(POSTGRES_DB_NAME).click();+ H.miniPicker().findByText(POSTGRES_INPUT_SCHEMA).click();+ H.miniPicker().findByText(POSTGRES_TARGET_TABLE_DISPLAY_NAME).click();+ H.visualize();+ H.assertQueryBuilderRowCount(ROW_COUNT);++ cy.log("the actual warehouse table lives under the workspace schema");+ H.queryWritableDB(+ `SELECT COUNT(*) AS count FROM "${POSTGRES_OUTPUT_SCHEMA}"."${POSTGRES_INPUT_SCHEMA}__${POSTGRES_TARGET_TABLE}"`,+ "postgres",+ ).then((result) => {+ expect(Number(result.rows[0].count)).to.eq(ROW_COUNT);+ });++ cy.log("leave the workspace through the UI");+ H.CurrentWorkspacePage.visit();+ H.CurrentWorkspacePage.leaveButton().click();+ H.LeaveWorkspaceModal.confirmButton().click();+ H.WorkspaceListPage.setupInstanceButton().should("be.visible");+ });+ });++ describe("mysql", () => {+ beforeEach(() => {+ H.restore("mysql-writable");+ cy.signInAsAdmin();+ H.activateToken("bleeding-edge");+ H.resetTestTable({ type: "mysql", table: MYSQL_SOURCE_TABLE });+ H.queryWritableDB(+ `DROP DATABASE IF EXISTS ${MYSQL_OUTPUT_DATABASE}`,+ "mysql",+ );+ H.queryWritableDB(`CREATE DATABASE ${MYSQL_OUTPUT_DATABASE}`, "mysql");+ H.resyncDatabase({ dbId: WRITABLE_DB_ID });+ });++ afterEach(() => {+ H.clearWorkspaceInstanceConfig();+ H.queryWritableDB(+ `DROP DATABASE IF EXISTS ${MYSQL_OUTPUT_DATABASE}`,+ "mysql",+ );+ });++ it("runs a transform, surfaces the remapping on the instance page, and queries the remapped table", () => {+ cy.log("set up the workspace");+ H.WorkspaceListPage.visit();+ H.WorkspaceListPage.setupInstanceButton().click();+ H.SetupWorkspaceModal.uploadConfig(MYSQL_CONFIG);+ H.SetupWorkspaceModal.setupButton().click();+ H.CurrentWorkspacePage.get().should("be.visible");++ cy.log("create and run a transform via the API");+ createAndRunTransform({+ sourceTable: MYSQL_SOURCE_TABLE,+ sourceSchema: null,+ targetTable: MYSQL_TARGET_TABLE,+ targetSchema: null,+ });++ cy.log("instance page shows the remapping for the transform target");+ H.CurrentWorkspacePage.visit();+ H.CurrentWorkspacePage.database(MYSQL_DB_NAME)+ .should("contain.text", MYSQL_TARGET_TABLE)+ .and(+ "contain.text",+ `${MYSQL_OUTPUT_DATABASE}/__${MYSQL_TARGET_TABLE}`,+ );++ cy.log("native query is rewritten to workspace table");+ H.startNewNativeQuestion({+ database: WRITABLE_DB_ID,+ query: `SELECT * FROM \`${MYSQL_TARGET_TABLE}\``,+ });+ H.runNativeQuery();+ H.assertQueryBuilderRowCount(ROW_COUNT);++ cy.log("mbql query is rewritten to workspace table");+ H.startNewQuestion();+ H.miniPicker().findByText(MYSQL_DB_NAME).click();+ H.miniPicker().findByText(MYSQL_TARGET_TABLE_DISPLAY_NAME).click();+ H.visualize();+ H.assertQueryBuilderRowCount(ROW_COUNT);++ cy.log("the actual warehouse table lives under the workspace database");+ H.queryWritableDB(+ `SELECT COUNT(*) AS count FROM \`${MYSQL_OUTPUT_DATABASE}\`.\`__${MYSQL_TARGET_TABLE}\``,+ "mysql",+ ).then((result) => {+ expect(Number(result.rows[0].count)).to.eq(ROW_COUNT);+ });++ cy.log("leave the workspace through the UI");+ H.CurrentWorkspacePage.visit();+ H.CurrentWorkspacePage.leaveButton().click();+ H.LeaveWorkspaceModal.confirmButton().click();+ H.WorkspaceListPage.setupInstanceButton().should("be.visible");+ });+ });+});++function createAndRunTransform({+ sourceTable,+ sourceSchema,+ targetTable,+ targetSchema,+}: {+ sourceTable: string;+ sourceSchema: string | null;+ targetTable: string;+ targetSchema: string | null;+}) {+ cy.log("create and run a transform via the API");+ H.getTableId({+ databaseId: WRITABLE_DB_ID,+ name: sourceTable,+ schema: sourceSchema ?? undefined,+ }).then((sourceTableId) => {+ H.createTestQuery({+ database: WRITABLE_DB_ID,+ stages: [+ {+ source: { type: "table", id: sourceTableId },+ limit: ROW_COUNT,+ },+ ],+ }).then((query) => {+ H.createTransform({+ name: "Workspace transform",+ source: { type: "query", query },+ target: {+ type: "table",+ database: WRITABLE_DB_ID,+ name: targetTable,+ schema: targetSchema,+ },+ }).then(({ body: transform }) => {+ H.runTransformAndWaitForSuccess(transform.id);+ });+ });+ });+}diff --git a/e2e/test/scenarios/workspaces/workspace-manager.cy.spec.ts b/e2e/test/scenarios/workspaces/workspace-manager.cy.spec.tsnew file mode 100644index 00000000000..407e3830b0d--- /dev/null+++ b/e2e/test/scenarios/workspaces/workspace-manager.cy.spec.ts@@ -0,0 +1,126 @@+import path from "path";++import { WRITABLE_DB_ID } from "e2e/support/cypress_data";++const { H } = cy;++const CONFIG_FILENAME = "config.yml";++describe("scenarios > workspaces > workspace manager", () => {+ describe("postgres (with schemas)", () => {+ const POSTGRES_DB_NAME = "Writable Postgres12";+ const PG_SCHEMA_A = "Domestic";+ const PG_SCHEMA_B = "Wild";+ const workspaceName = "PG Workspace";+ const renamedName = "Renamed PG Workspace";++ beforeEach(() => {+ H.restore("postgres-writable");+ cy.signInAsAdmin();+ H.activateToken("bleeding-edge");+ H.resetTestTable({ type: "postgres", table: "multi_schema" });+ H.resyncDatabase({ dbId: WRITABLE_DB_ID });+ enableWorkspaces(WRITABLE_DB_ID);+ cy.deleteDownloadsFolder();+ });++ it("creates, renames, downloads config, and deletes a workspace as admin", () => {+ cy.log(+ "create the workspace from the empty state, databases added at once",+ );+ H.WorkspaceListPage.visit();+ H.WorkspaceListPage.newButton().click();+ H.NewWorkspaceModal.nameInput().type(workspaceName);+ H.NewWorkspaceModal.databaseCheckbox(POSTGRES_DB_NAME).click();+ H.NewWorkspaceModal.createButton().click();++ cy.log("the database (with all of its schemas) is attached at once");+ H.WorkspaceListPage.workspace(workspaceName)+ .should("be.visible")+ .and("contain.text", POSTGRES_DB_NAME);++ cy.log("rename the workspace via the menu");+ H.WorkspaceListPage.workspaceMenuButton(workspaceName).click();+ H.WorkspaceListPage.renameMenuItem().click();+ H.RenameWorkspaceModal.nameInput().clear().type(renamedName);+ H.RenameWorkspaceModal.renameButton().click();+ H.WorkspaceListPage.workspace(renamedName).should("be.visible");++ cy.log("download the workspace config via the menu");+ H.WorkspaceListPage.workspaceMenuButton(renamedName).click();+ H.WorkspaceListPage.downloadConfigMenuItem().click();+ cy.verifyDownload(CONFIG_FILENAME, { timeout: 15000 });+ readConfig().should((contents) => {+ expect(contents).to.contain(renamedName);+ expect(contents).to.contain(POSTGRES_DB_NAME);+ expect(contents).to.contain("postgres");+ expect(contents).to.contain(PG_SCHEMA_A);+ expect(contents).to.contain(PG_SCHEMA_B);+ });++ cy.log("delete the workspace via the menu");+ H.WorkspaceListPage.workspaceMenuButton(renamedName).click();+ H.WorkspaceListPage.deleteMenuItem().click();+ H.DeleteWorkspaceModal.confirmButton().click();+ H.WorkspaceListPage.workspaceList().should("not.exist");+ H.WorkspaceListPage.newButton().should("be.visible");+ });+ });++ describe("mysql (no schemas)", () => {+ const MYSQL_DB_NAME = "Writable MySQL8";+ const workspaceName = "MySQL Workspace";++ beforeEach(() => {+ H.restore("mysql-writable");+ cy.signInAsAdmin();+ H.activateToken("bleeding-edge");+ enableWorkspaces(WRITABLE_DB_ID);+ cy.deleteDownloadsFolder();+ });++ it("creates, downloads config, and deletes a workspace as admin", () => {+ cy.log("create the workspace with the schemaless database");+ H.WorkspaceListPage.visit();+ H.WorkspaceListPage.newButton().click();+ H.NewWorkspaceModal.nameInput().type(workspaceName);+ H.NewWorkspaceModal.databaseCheckbox(MYSQL_DB_NAME).click();+ H.NewWorkspaceModal.createButton().click();++ H.WorkspaceListPage.workspace(workspaceName)+ .should("be.visible")+ .and("contain.text", MYSQL_DB_NAME);++ cy.log("download the workspace config");+ H.WorkspaceListPage.workspaceMenuButton(workspaceName).click();+ H.WorkspaceListPage.downloadConfigMenuItem().click();+ cy.verifyDownload(CONFIG_FILENAME, { timeout: 15000 });+ readConfig().should((contents) => {+ expect(contents).to.contain(workspaceName);+ expect(contents).to.contain(MYSQL_DB_NAME);+ expect(contents).to.contain("mysql");+ expect(contents).not.to.contain("schema-filters-patterns");+ });++ cy.log("delete the workspace");+ H.WorkspaceListPage.workspaceMenuButton(workspaceName).click();+ H.WorkspaceListPage.deleteMenuItem().click();+ H.DeleteWorkspaceModal.confirmButton().click();+ H.WorkspaceListPage.workspaceList().should("not.exist");+ H.WorkspaceListPage.newButton().should("be.visible");+ });+ });+});++function enableWorkspaces(databaseId: number) {+ cy.request("GET", `/api/database/${databaseId}`).then(({ body }) => {+ cy.request("PUT", `/api/database/${databaseId}`, {+ settings: { ...body.settings, "database-enable-workspaces": true },+ });+ });+}++function readConfig() {+ const downloadsFolder = Cypress.config("downloadsFolder");+ return cy.readFile(path.join(downloadsFolder, CONFIG_FILENAME));+}diff --git a/enterprise/backend/src/metabase_enterprise/advanced_config/api.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/api.cljindex 295a5b758d5..c5abcdd0e9c 100644--- a/enterprise/backend/src/metabase_enterprise/advanced_config/api.clj+++ b/enterprise/backend/src/metabase_enterprise/advanced_config/api.clj@@ -21,25 +21,6 @@metabase-enterprise.advanced-config.file.users/keep-memetabase-enterprise.advanced-config.file.workspace/keep-me)-(def ConfigUploadRequest- "Schema for the multipart `config.yml` upload request, shared with the- `/external` variant."- [:map- [:multipart-params- [:map- ["config" ms/File]]]])--(defn apply-config-upload!- "Apply the uploaded `config.yml` multipart param `config` to this instance and- delete its tempfile. Shared by the `/` and `/external` upload endpoints."- [config]- (let [tempfile (:tempfile config)]- (try- (advanced-config.file/initialize! (yaml/from-file tempfile))- (finally- (io/delete-file tempfile :silently))))- nil)-(api.macros/defendpoint :post "/" :- :nil"Apply an uploaded `config.yml` to this instance. Runs the same per-sectioninitializers (`settings`, `databases`, `users`, `api-keys`, `workspace`, ...)@@ -52,10 +33,17 @@[_route-params_query-params_body- {{config "config"} :multipart-params, :as _request} :- ConfigUploadRequest]+ {{config "config"} :multipart-params, :as _request}+ :- [:map+ [:multipart-params+ [:map+ ["config" [:map+ [:filename :string]+ [:tempfile (ms/InstanceOfClass java.io.File)]]]]]]](api/check-superuser)- (apply-config-upload! config))--(def ^{:arglists '([request respond raise])} routes- "`/api/ee/advanced-config` routes."- (api.macros/ns-handler *ns*))+ (let [tempfile (:tempfile config)]+ (try+ (advanced-config.file/initialize! (yaml/from-file tempfile))+ (finally+ (io/delete-file tempfile :silently))))+ nil)diff --git a/enterprise/backend/src/metabase_enterprise/advanced_config/api/external.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/api/external.cljdeleted file mode 100644index d061a4b4720..00000000000--- a/enterprise/backend/src/metabase_enterprise/advanced_config/api/external.clj+++ /dev/null@@ -1,26 +0,0 @@-(ns metabase-enterprise.advanced-config.api.external- "Static-API-key variant of the `config.yml` upload, served under- `/api/ee/advanced-config/external`. The route is wrapped in- `+static-apikey`, so requests authenticate with the static `MB_API_KEY`- (`x-metabase-apikey` header) instead of a user session — for out-of-band- automation, e.g. a workspace manager configuring a freshly provisioned- child instance."- (:require- [metabase-enterprise.advanced-config.api :as advanced-config.api]- [metabase.api.macros :as api.macros]))--(api.macros/defendpoint :post "/" :- :nil- "Apply an uploaded `config.yml` to this instance. Same behavior as- `POST /api/ee/advanced-config`, authenticated with the static `MB_API_KEY`- header instead of a superuser session."- {:multipart true}- [_route-params- _query-params- _body- {{config "config"} :multipart-params, :as _request}- :- advanced-config.api/ConfigUploadRequest]- (advanced-config.api/apply-config-upload! config))--(def ^{:arglists '([request respond raise])} routes- "`/api/ee/advanced-config/external` routes."- (api.macros/ns-handler *ns*))diff --git a/enterprise/backend/src/metabase_enterprise/advanced_config/api/routes.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/api/routes.cljdeleted file mode 100644index 56c3988f871..00000000000--- a/enterprise/backend/src/metabase_enterprise/advanced_config/api/routes.clj+++ /dev/null@@ -1,13 +0,0 @@-(ns metabase-enterprise.advanced-config.api.routes- "Routes for the EE advanced-config API endpoints."- (:require- [metabase-enterprise.advanced-config.api :as advanced-config.api]- [metabase-enterprise.advanced-config.api.external :as advanced-config.api.external]- [metabase.api.routes.common :refer [+static-apikey]]- [metabase.api.util.handlers :as handlers]))--(def ^{:arglists '([request respond raise])} routes- "Ring routes for the EE advanced-config API endpoints."- (handlers/route-map-handler- {"/" advanced-config.api/routes- "/external" (+static-apikey advanced-config.api.external/routes)}))diff --git a/enterprise/backend/src/metabase_enterprise/advanced_config/file.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/file.cljindex 9f362a7bf78..fa04e6961a3 100644--- a/enterprise/backend/src/metabase_enterprise/advanced_config/file.clj+++ b/enterprise/backend/src/metabase_enterprise/advanced_config/file.clj@@ -102,6 +102,7 @@[metabase-enterprise.advanced-config.file.settings][metabase-enterprise.advanced-config.file.users][metabase-enterprise.advanced-config.file.workspace :as advanced-config.file.workspace]+ [metabase-enterprise.workspaces.core :as ws][metabase.lib.core :as lib][metabase.premium-features.core :as premium-features][metabase.util :as u]@@ -293,6 +294,15 @@(defn boot-initialize!"Boot-time entry point: read the config file from disk and run [[initialize!]]- with `{{env VAR}}` template expansion enabled. No-op when no file is present."+ with `{{env VAR}}` template expansion enabled. No-op when no file is present.+ When the config contains a `:workspace` section, locks the workspace against+ runtime mutation (see+ [[metabase-enterprise.workspaces.core/workspace-locked-by-config?]])."[]- (initialize! (config-from-disk) {:expand-templates? true}))+ (let [parsed (config-from-disk)+ result (initialize! parsed {:expand-templates? true})]+ ;; Only the boot path locks. Runtime config uploads (POST /api/ee/advanced-config)+ ;; call `initialize!` directly, bypassing this wrapper, so they never lock.+ (when (get-in parsed [:config :workspace])+ (ws/mark-locked-by-config!))+ result))diff --git a/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.cljindex 9991c73aa38..e1960ec0560 100644--- a/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj+++ b/enterprise/backend/src/metabase_enterprise/advanced_config/file/databases.clj@@ -8,8 +8,8 @@[metabase.driver.util :as driver.u][metabase.sample-data.core :as sample-data][metabase.util :as u]- [metabase.util.jvm :as u.jvm][metabase.util.log :as log]+ [metabase.util.quick-task :as quick-task][toucan2.core :as t2]))(s/def :metabase-enterprise.advanced-config.file.databases.config-file-spec/name@@ -137,7 +137,7 @@(advanced-config.settings/config-from-file-sync-databases)(let [sync-database! (requiring-resolve 'metabase.sync.core/sync-database!)]- (u.jvm/in-virtual-thread* (sync-database! db)))+ (quick-task/submit-task! (fn [] (sync-database! db)))):else(log/info "Sync on database creation when initializing from file is disabled. Skipping sync."))))))))diff --git a/enterprise/backend/src/metabase_enterprise/advanced_config/file/users.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/file/users.cljindex 6f2bec1a019..b64dd23a92d 100644--- a/enterprise/backend/src/metabase_enterprise/advanced_config/file/users.clj+++ b/enterprise/backend/src/metabase_enterprise/advanced_config/file/users.clj@@ -10,10 +10,10 @@[toucan2.core :as t2]))(s/def :metabase-enterprise.advanced-config.file.users.config-file-spec/first_name- (s/nilable string?))+ string?)(s/def :metabase-enterprise.advanced-config.file.users.config-file-spec/last_name- (s/nilable string?))+ string?)(s/def :metabase-enterprise.advanced-config.file.users.config-file-spec/passwordstring?)diff --git a/enterprise/backend/src/metabase_enterprise/advanced_config/file/workspace.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/file/workspace.cljindex 29addbf616a..31be057e107 100644--- a/enterprise/backend/src/metabase_enterprise/advanced_config/file/workspace.clj+++ b/enterprise/backend/src/metabase_enterprise/advanced_config/file/workspace.clj@@ -15,9 +15,7 @@[clojure.walk :as walk][metabase-enterprise.advanced-config.file.interface :as advanced-config.file.i][metabase-enterprise.advanced-config.file.workspace.output :as-alias wkspc-output]- [metabase-enterprise.remote-sync.core :as remote-sync][metabase-enterprise.workspaces.core :as ws]- [metabase.util.jvm :as u.jvm][metabase.util.log :as log][toucan2.core :as t2]))@@ -118,7 +116,4 @@(defmethod advanced-config.file.i/initialize-section! :workspace[_section-name section-config]- (apply-workspace-section! section-config)- (when (remote-sync/remote-sync-enabled)- (log/info "Workspace instance with remote sync configured; starting the initial import")- (u.jvm/in-virtual-thread* (remote-sync/start-import!))))+ (apply-workspace-section! section-config))diff --git a/enterprise/backend/src/metabase_enterprise/api_routes/routes.clj b/enterprise/backend/src/metabase_enterprise/api_routes/routes.cljindex c7476deb667..f3cea35500d 100644--- a/enterprise/backend/src/metabase_enterprise/api_routes/routes.clj+++ b/enterprise/backend/src/metabase_enterprise/api_routes/routes.clj@@ -6,8 +6,8 @@`enterprise/backend/README.md` for more details."(:require[metabase-enterprise.action-v2.api]+ [metabase-enterprise.advanced-config.api][metabase-enterprise.advanced-config.api.logs]- [metabase-enterprise.advanced-config.api.routes][metabase-enterprise.advanced-permissions.api.routes][metabase-enterprise.api.core :as ee.api][metabase-enterprise.audit-app.api.routes]@@ -109,7 +109,7 @@routes here and follow the convention.";; Postponing a granular flag for :actions until it's used more widely.{"/action-v2" (premium-handler metabase-enterprise.action-v2.api/routes :table-data-editing)- "/advanced-config" metabase-enterprise.advanced-config.api.routes/routes+ "/advanced-config" (api.macros/ns-handler 'metabase-enterprise.advanced-config.api)"/advanced-permissions" (premium-handler metabase-enterprise.advanced-permissions.api.routes/routes :advanced-permissions)"/ai-controls" (premium-handler metabase-enterprise.metabot.api.routes/routes :ai-controls)"/audit-app" (premium-handler metabase-enterprise.audit-app.api.routes/routes :audit-app)diff --git a/enterprise/backend/src/metabase_enterprise/checker/README.md b/enterprise/backend/src/metabase_enterprise/checker/README.mddeleted file mode 100644index 7d2fd200201..00000000000--- a/enterprise/backend/src/metabase_enterprise/checker/README.md+++ /dev/null@@ -1,350 +0,0 @@-# Checker Module--Validates serdes YAML exports without a running Metabase instance or database connection.--Given two directories — database schemas and exported entities — the checker-resolves all references, runs query validation, and reports what's broken.--## Quick Start--```bash-# Dev mode-clj -M:dev:drivers:ee -m metabase.core.bootstrap \- --mode checker \- --export /path/to/export \- --schema-dir /path/to/databases--# Built jar-java -jar metabase.jar --mode checker \- --export /path/to/export --schema-dir /path/to/databases-```--Exit code 0 when all checks pass, 1 when any fail.--Example call against representations with edits intentionall introducing errors:--```-❯ clj -M:dev:drivers:ee -m metabase.core.bootstrap --mode checker --export /Users/dan/projects/work/representations/examples/v1 --schema-dir /Users/dan/projects/work/yaml-checked-files-v1/exports/sqlite-based/databases-[lots of logging removed]-Semantic Check Results-=====================-Total entities: 79- OK: 76- Errors: 0- Unresolved refs: 1- Native SQL errors: 2- Issues: 0--Failures:------------=== Daily Order Summary [7XqqzlsonSq4KFyOvtrAm] ===- Kind: transform- UNRESOLVED REFERENCES:- NATIVE SQL ERRORS:- - {:type :syntax-error}- SQL: SELECT- DATE_TRUNC('day', CREATED_AT) AS order_date- COUT(*) AS order_count,- SU(TOTAL) AS total_revenue-FROM ORDERS-GROUP BY 1- Status: NATIVE SQL ERRORS--=== Native Query - Card and Snippet References [HiBFSt0BNx5s5MxVDLLKB] ===- Card ID: 94- Tables: Sample Database.PUBLIC.ORDERS- Fields: Sample Database.PUBLIC.ORDERS.CREATED_AT- Source Cards: Basic Aggregations- NATIVE SQL ERRORS:- - {:type :missing-column, :name "created_attt", :source-entity-type :card, :source-entity-id "h5F2EjHsRd73Dqqh8sAtd"}- - {:type :missing-column, :name "cooount", :source-entity-type :card, :source-entity-id "h5F2EjHsRd73Dqqh8sAtd"}- Status: NATIVE SQL ERRORS--=== Product Analysis Report [dOc1PrOdAnAlYsIsRpTx2] ===- Kind: document- UNRESOLVED REFERENCES:- - document-link: Q_jD-f-9clKLFZ2TfUG2g- Status: MISSING REFS--❯ echo $?-1-```--## What It Validates--### Query validation (via `deps.analysis/check-entity`)--- **MBQL**: bad field/table/card references, broken joins-- **Native SQL**: missing columns, syntax errors, bad table references-- **Transforms**: query validation plus duplicate output column detection-- Source attribution on errors (which table or card caused the problem)--### Structural checks--- `collection_id`, `dashboard_id`, `document_id`, `parent_id` point to entities of the correct kind-- Dashboard layout: card refs exist, tab refs match, grid positions in bounds-- Document content: embedded card refs and entity links resolve-- Transform database refs exist in the schema-- Duplicate `entity_id`s across files--### Dependency tracking--Each card result includes a `:refs` map showing what it touches:--- `:tables` — database tables referenced-- `:fields` — specific fields used-- `:source-cards` — other cards used as data sources-- `:snippets` — native query snippets used-- `:measures`, `:metrics`, `:segments` — referenced by entity-id--These propagate transitively through source-card chains.--## CLI Options--```---export PATH Path to serdes export directory (collections/)---schema-dir PATH Path to database schema directory (databases/)---output PATH Write raw results to a file---errors-only Output only errors to stdout (concise format)---help Show help-```--`--errors-only` outputs one block per failing entity:--```-card: Number of Orders (entity_id: 8EdazRgPwfxdiltp7NCjS)- unresolved field: Sample Database.PUBLIC.ORDERS.STARTED_AT-transform: Bad Transform (entity_id: 6Fdv3rO4bB5xyXusrVEGS)- unresolved field: zomato..zomato.deliveryyyy-```--## Running Tests--```bash-clj -X:dev:test:ee:ee-dev :module enterprise/checker-```--Tests include unit tests (in-memory sources), integration tests (YAML fixtures),-and e2e tests against the baseline export at `test_resources/serialization_baseline`.--## REPL Usage--```clojure-(require '[metabase-enterprise.checker.semantic :as checker])--;; Check the baseline export-(def results- (:results (checker/check "test_resources/serialization_baseline"- "test_resources/serialization_baseline/databases")))--(checker/summarize-results results)-;; => {:total 80, :ok 80, :errors 0, :unresolved 0, :native-errors 0, :issues 0}--;; Inspect specific entities-(select-keys results ["wpW9JO5MsHQd0-7WrR7gN"- "kBjQ5VXJ5z3vYSW72J6qa"- "dOc1PrOdAnAlYsIsRpTx2"])-;; =>-;; {"wpW9JO5MsHQd0-7WrR7gN"-;; {:card-id 67,-;; :name "Joins - Strategies and Compound Conditions",-;; :entity-id "wpW9JO5MsHQd0-7WrR7gN",-;; :refs {:tables ["Sample Database.PUBLIC.PRODUCTS"-;; "Sample Database.PUBLIC.REVIEWS"-;; "Sample Database.PUBLIC.ORDERS"-;; "Sample Database.PUBLIC.PEOPLE"],-;; :fields ["Sample Database.PUBLIC.ORDERS.ID"-;; "Sample Database.PUBLIC.ORDERS.USER_ID"-;; ...19 fields across 4 tables...]}},-;;-;; "kBjQ5VXJ5z3vYSW72J6qa"-;; {:card-id 6,-;; :name "Metric, Measure, and Segment References",-;; :entity-id "kBjQ5VXJ5z3vYSW72J6qa",-;; :refs {:tables ["Sample Database.PUBLIC.ORDERS"],-;; :source-cards ["Total Revenue"],-;; :measures ["DBap2523KuN4Lt-cYrUjF"],-;; :metrics ["IW8kbqZVaMxdGtCM2F4U6"],-;; :segments ["OaFXAvEzPvu9ZXW5PUMRa"]}},-;;-;; "dOc1PrOdAnAlYsIsRpTx2"-;; {:name "Product Analysis Report",-;; :entity-id "dOc1PrOdAnAlYsIsRpTx2",-;; :kind :document}}--;; Interactive iteration on a single card-(def ctx (checker/setup "test_resources/serialization_baseline"- "test_resources/serialization_baseline/databases"))-(checker/check-one ctx "HiBFSt0BNx5s5MxVDLLKB" :verbose true)-```--## Architecture--```- SchemaSource ──┐ ┌── query validation (deps.analysis)- (databases, │ │- tables, ├─→ Store ─→ Provider ─→ MetadataProvider- fields) │ (IDs, (lib │- │ cache) adapter) └── structural checks- AssetsSource ──┘ (collection refs, dashboard- (cards, layout, document links)- snippets,- transforms,- segments)-```--1. **Sources** provide entity data. `SchemaSource` resolves databases/tables/fields- from `--schema-dir`. `AssetsSource` resolves cards/snippets/transforms/segments- from `--export`. Both load YAML lazily.-2. **Store** combines both sources with a file index. Knows what entities exist,- loads them on demand, assigns synthetic integer IDs (lib requires ints), caches- everything.-3. **Provider** adapts the store into a `MetadataProvider` that `lib/query`- understands. Converts raw YAML to lib metadata format, resolves portable refs- to integer IDs, normalizes MBQL. Owns a mutable `current-db` atom set per- entity check.-4. **Validation** runs query checks via `deps.analysis/check-entity` and- structural checks on entity relationships.--### On-demand indexing--The schema directory can contain 500k+ field files. To avoid parsing them all-at startup:--- **Databases** (~40): indexed at startup by reading only the `name:` field- from each database YAML via fast regex (no full parse).-- **Tables** (~22k): enumerated on demand by listing directories. Assigned- lightweight IDs from directory paths via `ensure-table-id!` (no YAML parse).- Full YAML is parsed only when table data is actually needed.-- **Fields** (~500k): resolved on demand from `<table>/fields/` directories.- Never pre-indexed.-- **Segments/measures**: indexed from the export dir only (walks table- directories for `segments/` and `measures/` subdirs). Not indexed in the- schema dir.--Directory names are slugified (e.g. `analytics_data_warehouse` for-"Analytics Data Warehouse"). A `db-name->dir` mapping built from database-YAMLs handles the translation. Schema and table directory names match their-real names.--This brings setup time from ~64s to ~100ms.--### Running without an app DB--The checker runs the full QP compilation pipeline (template tag expansion,-snippet inlining, card ref resolution) but without a running Metabase instance.-Two things make this possible:--- **`qp.i/*skip-middleware-because-app-db-access*`** is bound to `true`, which- skips middleware that queries the app DB for access control. Currently skipped:- - **Impersonation** — connection role swapping based on user permissions- - **Database routing** — routing queries to alternative destination databases-- These are security middleware, not semantic middleware. They control *who runs- where*, not the shape of the query. Skipping them is safe for validation.--- **`mu.fn/*enforce*`** is bound to `false`, disabling Malli output schema- enforcement so `lib/query` accepts valid MBQL clauses that the schemas don't- fully enumerate.--This means native SQL validation resolves through `{{#card-ref}}` and-`{{snippet: Name}}` template tags using real data from the store, catching-missing columns with source attribution back to the originating card or snippet.--### Namespaces--- **`checker.source`** — `SchemaSource` and `AssetsSource` protocols-- **`checker.format.serdes`** — serdes directory walking, YAML extraction, file index building, `SerdesSource` (implements both protocols)-- **`checker.store`** — combines sources + index, ID registry, entity caches, lazy loading-- **`checker.provider`** — adapts store to lib `MetadataProvider`, data conversion, MBQL normalization, reference resolution-- **`checker.semantic`** — entity validation, result formatting, top-level API (`check`, `setup`, `check-one`)-- **`checker.cli`** — CLI entrypoint and argument parsing--## Performance--- **Setup**: ~100ms (reads ~40 database YAMLs)-- **Per entity**: ~1s average, cached after first access per database-- **First entity per database**: ~5-8s (table directory listing + ID assignment)-- **Complex native SQL**: 5-20s per entity (SQLGlot parsing bottleneck)-- **Full run** (177 entities, 40 databases, 22k tables): ~90s--### Parallelism--The architecture is ready for multi-threaded entity checking:-- The store (atom) is thread-safe-- Sources are stateless/immutable-- Each thread needs its own `provider` (owns mutable `current-db`)-- Dynamic bindings (`mu.fn/*enforce*`, `qp.i/*skip-middleware*`) are per-thread--To parallelize: create a provider per thread, replace the `for` in-`check-entities` with `pmap` or a thread pool.--## Roadmap--### Dependency graph--The checker computes forward edges per entity (what it depends on). The next-step is building a bidirectional dependency graph using-`metabase.graph.core/InMemoryGraph`.--The `dependencies` module already has the right pieces:--- **`calculation.clj`** — `upstream-deps:card`, `upstream-deps:transform`, etc.- extract forward edges from resolved lib queries. These just call- `lib/all-source-table-ids` etc., which work fine against our provider.-- **`analysis.clj`** — `check-entity` validates individual entities. The checker- already uses this.-- **`metabase.graph.core`** — generic `Graph` protocol with `InMemoryGraph`,- `transitive`, `find-cycle`, etc.--The plan:--1. Call `calculation.clj`'s `upstream-deps:*` functions against the checker's- resolved entities (or align our `extract-refs-from-query` output to match)-2. Collect all edges into an adjacency map, wrap in `graph/in-memory`-3. Invert the map for the reverse graph-4. Return both from `checker/check`--This enables "who uses this snippet?", "what breaks if I rename this table?",-cycle detection via `graph/find-cycle`, and full impact analysis — all offline.--### Decouple dependencies module from app DB--`deps.analysis` and `deps.calculation` are clean — they operate on-MetadataProviders and lib queries with no toucan2 dependency. But-`deps.core` and `deps.models.dependency` are tightly coupled to the app DB-(toucan2, `:model/Dependency` table, etc.).--Currently the checker imports `deps.analysis` directly, avoiding the toucan2-dependency chain. For the graph work, we'll also need `deps.calculation`-(also clean) and `metabase.graph.core` (also clean).--The goal is that the "pure analysis" side of the dependencies module-(`analysis.clj`, `calculation.clj`) stays usable without an app DB connection,-while the "storage" side (`models/dependency.clj`, `events.clj`) stays in leaf-namespaces that the checker never touches. This mirrors the pattern in-`models.serialization` where we separated the resolve protocols from the-DB-backed implementations.--### Merge native errors into bad-refs--`:native-errors` (from SQL parsing) and `:bad-refs` (from MBQL analysis) are-both query issues found through different code paths. Consider merging them-into a single `:query-issues` key. The other result keys (`:error` for fatal-failures, `:unresolved` for missing schema refs) are genuinely different-categories and should stay separate.--### Table-type template tag validation--Native queries with `{{table_one}}`-style template tags (type: table) currently-get placeholder substitution — column validation is skipped because the actual-table is unknown. The template tag contains a `table-id` that could be resolved-via the provider to substitute the real table name, enabling full column-validation against the schema.--### Schema mismatch false positives--`sql-tools/validate-query` defaults unqualified tables to `default-schema`,-which returns `"public"` for schema-less databases where tables have `nil`-schema. Root cause is in `sql-tools`, not the checker.diff --git a/enterprise/backend/src/metabase_enterprise/checker/cli.clj b/enterprise/backend/src/metabase_enterprise/checker/cli.cljdeleted file mode 100644index 9aee40529f8..00000000000--- a/enterprise/backend/src/metabase_enterprise/checker/cli.clj+++ /dev/null@@ -1,141 +0,0 @@-(ns metabase-enterprise.checker.cli- "CLI entrypoint for the checker module.-- Invoked from metabase.core.bootstrap:- java -jar metabase.jar --mode checker --export /path/to/export --schema-dir /path/to/schemas"- (:require- [clojure.java.io :as io]- [clojure.string :as str]- [clojure.tools.cli :as cli]- [metabase-enterprise.checker.semantic :as checker]- [metabase.classloader.core :as classloader]- [metabase.plugins.core :as plugins]))--(set! *warn-on-reflection* true)--#_{:clj-kondo/ignore [:discouraged-var]}-(def output!- "Alias for println so can suppress warning in one place"- println)--(defn- fail!- "Print error message to stderr and exit with code 1."- [& messages]- (binding [*out* *err*]- (doseq [msg messages]- (output! msg)))- (flush)- (System/exit 1))--(defn- validate-directory!- "Validate that path exists and is a directory. Exits on failure."- [path]- (let [f (io/file path)]- (cond- (not (.exists f))- (fail! (str "Directory does not exist: " path))-- (not (.isDirectory f))- (fail! (str "Not a directory: " path)))))--(defn- validate-schema-path!- "Validate that --schema-dir matches the expected shape for `fmt`:- :serdes → must be a directory (containing database subdirectories)- :concise → must be a regular file (the JSON metadata file)"- [path fmt]- (let [f (io/file path)]- (cond- (not (.exists f))- (fail! (str "--schema-dir does not exist: " path))-- (and (= fmt :serdes) (not (.isDirectory f)))- (fail! (str "--schema-format serdes expects a directory of database folders, got a file: " path)- "Did you mean --schema-format concise?")-- (and (= fmt :concise) (not (.isFile f)))- (fail! (str "--schema-format concise expects a single JSON file, got a directory: " path)- "Did you mean --schema-format serdes?"))))--(defn- print-human- "Print one FAIL block per failing entity, then a one-line summary."- [results]- (let [failures (->> results- (filter #(not= :ok (checker/result-status (second %))))- (sort-by (comp :name second)))]- (doseq [entry failures- :let [block (checker/format-fail entry)]- :when block]- (output! block)- (output!))- (output! (format "Ran %d entities, %d failed" (count results) (count failures)))))--(defn- run-checker- "Run the semantic checker."- [export-dir {:keys [output schema-dir schema-format]}]- (when-not schema-dir- (fail! "Missing --schema-dir option"))- (let [schema-fmt (case schema-format- "concise" :concise- "serdes" :serdes- nil :serdes- (fail! (str "Unknown --schema-format: " schema-format- " (must be 'serdes' or 'concise')")))- _ (validate-schema-path! schema-dir schema-fmt)- {:keys [results]} (try- (checker/check export-dir schema-dir nil {:schema-format schema-fmt})- (catch clojure.lang.ExceptionInfo e- (fail! (.getMessage e))))- summary (checker/summarize-results results)]- (when output- (spit output (pr-str results)))- (print-human results)- (flush)- (System/exit (if (zero? (+ (:errors summary) (:unresolved summary)- (:native-errors summary) (:issues summary)))- 0- 1))))--(def ^:private cli-spec- "CLI options for checker mode."- [[nil "--mode MODE" "Mode (handled by bootstrap, included here for completeness)"]- [nil "--export PATH" "Path to serdes export directory"]- [nil "--schema-dir PATH" "Path to schema source (directory for serdes, JSON file for concise)"]- [nil "--schema-format FMT" "Schema format: 'serdes' (default) or 'concise'"]- [nil "--output PATH" "Path to output file for raw EDN results"]- ["-h" "--help" "Show this help"]])--(defn- usage [summary]- (str "Usage: java -jar metabase.jar --mode checker [options]\n\n"- "Options:\n"- summary))--(defn entrypoint- "Main entrypoint for checker mode. Receives raw args. Intended to enter from- metabase.core.bootstrap/run-standalone-mode so that it can skip loading everything that- metabase.core.core/entrypoint loads. This entrypoint owns the process and will call System/exit."- [args]- (let [{:keys [options errors summary]} (cli/parse-opts args cli-spec)- {:keys [export help]} options]- (cond- help- (do (output! (usage summary))- (System/exit 0))-- errors- (fail! (str "Error parsing arguments:\n" (str/join "\n" errors))- ""- (usage summary))-- (not export)- (fail! "Missing --export option"- ""- (usage summary))-- :else- (do- (validate-directory! export)- ;; Drivers are jar-loaded plugins; install the dynamic classloader and- ;; load them so query validation can resolve driver multimethods.- (classloader/the-classloader)- (plugins/load-plugins!)- (run-checker export options)))))diff --git a/enterprise/backend/src/metabase_enterprise/checker/format/concise_schema.clj b/enterprise/backend/src/metabase_enterprise/checker/format/concise_schema.cljdeleted file mode 100644index b553e8b9ec1..00000000000--- a/enterprise/backend/src/metabase_enterprise/checker/format/concise_schema.clj+++ /dev/null@@ -1,118 +0,0 @@-(ns metabase-enterprise.checker.format.concise-schema- "Concise JSON schema format — single file with three flat lists, fully- in-memory.-- The file is a single JSON document with three flat lists keyed by integer- IDs. Tables reference their database via `:db_id`; fields reference their- table via `:table_id`:-- {\"databases\": [{\"id\": 1, \"name\": \"Sample Database\", \"engine\": \"h2\"}, ...],- \"tables\": [{\"id\": 3, \"db_id\": 1, \"name\": \"PRODUCTS\", \"schema\": \"PUBLIC\"}, ...],- \"fields\": [{\"id\": 12, \"table_id\": 3, \"name\": \"ID\",- \"base_type\": \"type/BigInteger\",- \"semantic_type\": \"type/PK\",- \"database_type\": \"BIGINT\"}, ...]}-- At load time we build path-keyed indexes by joining `tables.db_id →- databases.id` and `fields.table_id → tables.id`, so the rest of the- checker (which speaks portable paths like `[db-name schema table]`) sees- the same shape it gets from any other source.-- This replaces the 500k-file serdes directory tree."- (:require- [clojure.java.io :as io]- [metabase-enterprise.checker.source :as source]- [metabase.util.json :as json])- (:import- (java.io File)))--(set! *warn-on-reflection* true)--(defn- build-model- "Parse the concise metadata JSON file and build path-keyed indexes.-- The on-disk shape uses integer IDs to link entities. Here we resolve those- IDs to their containing entity's name/path so callers can use portable- references throughout.-- Returns:- {:databases {db-name db-data}- :tables {[db-name schema table-name] table-data}- :fields {[db-name schema table-name field-name] field-data}}"- [^File f]- (let [parsed (json/decode (io/input-stream f) true)- databases (:databases parsed)- tables (:tables parsed)- fields (:fields parsed)- ;; id-keyed lookup tables- db-by-id (into {} (map (juxt :id identity)) databases)- tbl-by-id (into {} (map (juxt :id identity)) tables)- ;; path-keyed result maps- db-map (into {} (map (juxt :name identity)) databases)- tbl-map (into {}- (keep (fn [t]- (when-let [db (db-by-id (:db_id t))]- (let [db-name (:name db)- schema (:schema t)- table-name (:name t)- table-path [db-name schema table-name]]- [table-path (assoc t :db_id db-name)]))))- tables)- field-map (into {}- (keep (fn [field]- (when-let [t (tbl-by-id (:table_id field))]- (when-let [db (db-by-id (:db_id t))]- (let [db-name (:name db)- schema (:schema t)- table-name (:name t)- field-name (:name field)- table-path [db-name schema table-name]- field-path [db-name schema table-name field-name]]- [field-path (assoc field :table_id table-path)])))))- fields)]- {:databases db-map- :tables tbl-map- :fields field-map}))--(defn- fields-by-table- "Build a map of table-path → set of field-paths from the fields map."- [fields-map]- (reduce-kv (fn [m field-path _]- (let [table-path (subvec field-path 0 3)]- (update m table-path (fnil conj #{}) field-path)))- {}- fields-map))--(deftype ConciseSchemaSource [databases tables fields fields-index]- source/SchemaSource- (resolve-database [_ db-name]- (get databases db-name))-- (resolve-table [_ table-path]- (get tables table-path))-- (resolve-field [_ field-path]- (get fields field-path))-- (fields-for-table [_ table-path]- (get fields-index table-path))-- (all-field-paths [_]- (set (keys fields)))-- (all-database-names [_]- (keys databases))-- (all-table-paths [_]- (keys tables))-- (tables-for-database [_ db-name]- (filterv #(= (first %) db-name) (keys tables))))--(defn make-source- "Create a SchemaSource from a single concise metadata JSON file.- Loads everything into memory at construction time."- [schema-file]- (let [{:keys [databases tables fields]} (build-model (io/file schema-file))- fi (fields-by-table fields)]- (->ConciseSchemaSource databases tables fields fi)))diff --git a/enterprise/backend/src/metabase_enterprise/checker/format/serdes_assets.clj b/enterprise/backend/src/metabase_enterprise/checker/format/serdes_assets.cljdeleted file mode 100644index 75401d6d07a..00000000000--- a/enterprise/backend/src/metabase_enterprise/checker/format/serdes_assets.clj+++ /dev/null@@ -1,304 +0,0 @@-(ns metabase-enterprise.checker.format.serdes-assets- "Serdes export format — directory structure, path parsing, entity loading.-- Directory structure (on disk, names may be slugified):- databases/<db-dir>/<db-dir>.yaml- databases/<db-dir>/schemas/<schema-dir>/tables/<table-dir>/<table-dir>.yaml- databases/<db-dir>/schemas/<schema-dir>/tables/<table-dir>/fields/<field>.yaml- databases/<db-dir>/tables/<table-dir>/<table-dir>.yaml # schema-less- databases/<db-dir>/tables/<table-dir>/fields/<field>.yaml # schema-less- collections/<entity-id>_<slug>/<entity-id>_<slug>.yaml- collections/.../cards/<entity-id>_<slug>.yaml- collections/.../dashboards/<entity-id>_<slug>.yaml-- Directory names may differ from entity names (e.g. `analytics_data_warehouse`- for \"Analytics Data Warehouse\", or `orders` for \"ORDERS\" in H2).-- ## Lazy resolution cache-- The `SerdesSource` maintains a single lazily-populated cache atom that maps- real entity names (from YAML) to file locations:-- {db-name ; real name, e.g. \"Analytics Data Warehouse\"- {schema ; real name or nil for schema-less- ::not-indexed ; schema exists but tables not yet walked- | {table-name ; real name from YAML- {:table-file \"path/to/table.yaml\" ; path string to the table YAML- :fields ::not-indexed ; fields not yet walked for this table- | {field-name \"path\" ...}}}}} ; real name → field YAML path-- Two levels of laziness:- 1. Schema → tables: realized when any table in that schema is resolved- (via `resolve-table`, `resolve-field`, `fields-for-table`) or when- `tables-for-database` is called. Walks the schema's `tables/` directory- and reads each table's `name:` from YAML via fast regex.- 2. Table → fields: realized when `resolve-field` or `fields-for-table` is- called for that specific table. Walks the table's `fields/` directory- and reads each field's `name:` via fast regex. Other tables in the same- schema are unaffected.-- The `db-name->dir` map (built at startup from ~40 database YAMLs) handles the- database name → directory name translation. Schema and table name → directory- name translation is handled by case-insensitive directory matching during- schema indexing."- (:require- [clojure.java.io :as io]- [clojure.string :as str]- [metabase-enterprise.checker.source :as source]- [metabase.util.yaml :as yaml])- (:import- (java.io File)))--(set! *warn-on-reflection* true)--;;; ===========================================================================-;;; YAML Loading-;;; ===========================================================================--(defn load-yaml- "Load and parse a YAML file."- [path]- (yaml/parse-string (slurp path)))--(defn- db-dir- "Resolve a real database name to its directory on disk."- ^File [^File databases-dir db-name->dir db-name]- (let [dir-name (get db-name->dir db-name db-name)]- (io/file databases-dir dir-name)))--(defn quick-extract- "Extract a field from YAML using regex (fast) with full parse fallback."- [file-path field-name pattern]- (try- (let [content (slurp file-path)]- (if-let [[_ value] (re-find pattern content)]- (str/trim value)- (get (yaml/parse-string content) (keyword field-name))))- (catch Exception _ nil)))--(defn- read-yaml-name- "Read the `name:` field from a YAML file using regex (fast).- Falls back to `fallback` if the file doesn't exist or can't be read."- [^File yaml-file ^String fallback]- (if (.exists yaml-file)- (or (quick-extract (.getPath yaml-file) "name" #"(?m)^name:\s*(.+)")- fallback)- fallback))--(defn extract-entity-id- "Extract the entity id from a yaml file without parsing it. Looking for a top level entity_id at start of line."- [file-path]- (quick-extract file-path "entity_id" #"(?m)^entity_id:\s*(\S+)"))--(defn extract-model- "Extract the serdes model from a YAML file (e.g., \"Collection\", \"Card\", \"Dashboard\").- Looks for the model field inside serdes/meta to avoid matching embedded refs."- [file-path]- (try- (let [content (slurp file-path)]- (when-let [[_ model] (re-find #"(?m)^serdes/meta:\s*\n\s*- id:.*\n\s+(?:label:.*\n\s+)?model:\s*(\S+)" content)]- (str/trim model)))- (catch Exception _ nil)))--(defn- index-segments-and-measures- "Walk a table directory's segments/ and measures/ subdirectories.- Reads entity_id from each YAML (only a handful exist).- Returns a seq of {:kind :ref :file} entries."- [^File table-dir]- (into []- (mapcat- (fn [[subdir kind]]- (let [^File d (io/file table-dir subdir)]- (when (.isDirectory d)- (keep (fn [^File f]- (when (and (.isFile f) (str/ends-with? (.getName f) ".yaml"))- (when-let [eid (extract-entity-id (.getPath f))]- {:kind kind :ref eid :file (.getPath f)})))- (.listFiles d))))))- [["segments" :segment] ["measures" :measure]]))--(defn- list-table-dirs- "Return a seq of table directory Files for a database.- Walks schemas/<schema>/tables/<table>/ and tables/<table>/ directories."- [^File databases-dir db-name->dir-map db-name]- (let [^File db (db-dir databases-dir db-name->dir-map db-name)- result (volatile! [])]- (let [^File schemas-dir (io/file db "schemas")]- (when (.isDirectory schemas-dir)- (doseq [^File schema-d (.listFiles schemas-dir)- :when (.isDirectory schema-d)]- (let [^File tables-dir (io/file schema-d "tables")]- (when (.isDirectory tables-dir)- (doseq [^File td (.listFiles tables-dir)- :when (.isDirectory td)]- (vswap! result conj td)))))))- (let [^File tables-dir (io/file db "tables")]- (when (.isDirectory tables-dir)- (doseq [^File td (.listFiles tables-dir)- :when (.isDirectory td)]- (vswap! result conj td))))- @result))--(defn- index-schema-dir- "Walk a databases directory and index databases, segments, and measures.- Reads ~40 database YAMLs for real names, plus a handful of segment/measure YAMLs- for entity_ids. Tables and fields are resolved on demand from the filesystem.-- When `include-segments-measures?` is true, also walks table directories- to find segment and measure YAMLs (for export dirs). Set to false for- schema-only dirs to avoid walking thousands of table directories.-- Returns {:entries [...]- :db-name->dir {real-db-name dir-name}}."- [schema-dir & {:keys [include-segments-measures?]}]- (let [schema-dir (io/file schema-dir)- db-name->dir (volatile! {})- entries (volatile! [])]- (doseq [^File db-dir (.listFiles schema-dir)- :when (.isDirectory db-dir)]- (let [dir-name (.getName db-dir)- db-yaml (io/file db-dir (str dir-name ".yaml"))- db-name (read-yaml-name db-yaml dir-name)]- (vswap! db-name->dir assoc db-name dir-name)- (vswap! entries conj {:kind :database :ref db-name- :file (if (.exists db-yaml) (.getPath db-yaml) (.getPath db-dir))})- ;; Walk table directories for segments and measures (export dirs only)- (when include-segments-measures?- (doseq [^File table-dir (list-table-dirs schema-dir @db-name->dir db-name)]- (vswap! entries into (index-segments-and-measures table-dir))))))- {:entries @entries- :db-name->dir @db-name->dir}))--;;; ===========================================================================-;;; File Walking - Build index of all entities-;;; ===========================================================================--(def ^:private model->kind- "Map serdes model names to index kinds."- {"Card" :card- "Dashboard" :dashboard- "Collection" :collection- "Document" :document- "Measure" :measure- "Segment" :segment- "NativeQuerySnippet" :snippet- "Transform" :transform})--(defn- index-collections-tree- "Walk the collections/ directory and index all entities by their serdes model.- Returns entries for cards, dashboards, and collections."- [export-dir]- (let [collections-dir (io/file export-dir "collections")]- (when (.isDirectory collections-dir)- (for [^File file (file-seq collections-dir)- :when (.isFile file)- :let [path (.getPath file)]- :when (str/ends-with? (.getName file) ".yaml")- :let [model (extract-model path)- kind (get model->kind model)]- :when kind- :let [entity-id (extract-entity-id path)]- :when entity-id]- {:kind kind :ref entity-id :file path}))))--(defn build-file-index- "Build index of all entity files in an export directory.- Returns `{:index {kind {ref file-path}} :db-name->dir {real-name dir-name}}`- where index also has `:duplicates` for refs appearing in multiple files."- [export-dir]- (let [databases-dir (io/file export-dir "databases")- schema-result (when (.isDirectory databases-dir)- (index-schema-dir databases-dir :include-segments-measures? true))- entries (concat- (:entries schema-result)- (index-collections-tree export-dir))- ;; Group by [kind ref] to find duplicates- by-key (reduce (fn [m {:keys [kind ref file]}]- (update m [kind ref] (fnil conj []) file))- {}- entries)- ;; Build the index (first file wins) and collect duplicates- index (reduce-kv (fn [idx [kind ref] files]- (assoc-in idx [kind ref] (first files)))- {}- by-key)- dupes (into []- (keep (fn [[[kind ref] files]]- (when (> (count files) 1)- {:kind kind :ref ref :files files})))- by-key)]- (cond-> index- (seq dupes) (assoc :duplicates dupes))))--(defn index-stats- "Get statistics about a file index.- Note: tables and fields are resolved on demand and won't appear in the index- for schema-dir sources. They will be 0 for `build-database-dir-index` results."- [index]- {:databases (count (:database index))- :tables (count (:table index))- :fields (count (:field index))- :measures (count (:measure index))- :segments (count (:segment index))- :cards (count (:card index))- :dashboards (count (:dashboard index))- :collections (count (:collection index))- :database-names (vec (keys (:database index)))})--(deftype SerdesSource [databases-dir assets-index]- source/AssetsSource- (resolve-card [_ entity-id]- (when-let [file (get-in assets-index [:card entity-id])]- (load-yaml file)))-- (resolve-snippet [_ entity-id]- (when-let [file (get-in assets-index [:snippet entity-id])]- (load-yaml file)))-- (resolve-transform [_ entity-id]- (when-let [file (get-in assets-index [:transform entity-id])]- (load-yaml file)))-- (resolve-segment [_ entity-id]- (when-let [file (get-in assets-index [:segment entity-id])]- (load-yaml file)))-- (resolve-dashboard [_ entity-id]- (when-let [file (get-in assets-index [:dashboard entity-id])]- (load-yaml file)))-- (resolve-collection [_ entity-id]- (when-let [file (get-in assets-index [:collection entity-id])]- (load-yaml file)))-- (resolve-document [_ entity-id]- (when-let [file (get-in assets-index [:document entity-id])]- (load-yaml file)))-- (resolve-measure [_ entity-id]- (when-let [file (get-in assets-index [:measure entity-id])]- (load-yaml file))))--(defn make-source- "Create a MetadataSource for a serdes export directory.- The databases-dir for field resolution is export-dir/databases."- [export-dir]- (let [databases-dir (io/file export-dir "databases")- index (build-file-index export-dir)- assets-index (dissoc index :database)]- (->SerdesSource databases-dir assets-index)))--(defn source-index- "Get the assets index from a SerdesSource. Returns a flat map of- {[kind ref] file-path} for cards, dashboards, segments, etc."- [^SerdesSource source]- (.-assets-index source))--;;; ===========================================================================-;;; Enumeration-;;; ===========================================================================--(defn all-card-ids- "Get all card entity-ids from source."- [^SerdesSource source]- (keys (:card (.-assets-index source))))diff --git a/enterprise/backend/src/metabase_enterprise/checker/format/serdes_schema.clj b/enterprise/backend/src/metabase_enterprise/checker/format/serdes_schema.cljdeleted file mode 100644index d3d7d5652d2..00000000000--- a/enterprise/backend/src/metabase_enterprise/checker/format/serdes_schema.clj+++ /dev/null@@ -1,375 +0,0 @@-(ns metabase-enterprise.checker.format.serdes-schema- "Serdes export format — directory structure, path parsing, entity loading.-- Directory structure (on disk, names may be slugified):- databases/<db-dir>/<db-dir>.yaml- databases/<db-dir>/schemas/<schema-dir>/tables/<table-dir>/<table-dir>.yaml- databases/<db-dir>/schemas/<schema-dir>/tables/<table-dir>/fields/<field>.yaml- databases/<db-dir>/tables/<table-dir>/<table-dir>.yaml # schema-less- databases/<db-dir>/tables/<table-dir>/fields/<field>.yaml # schema-less- collections/<entity-id>_<slug>/<entity-id>_<slug>.yaml- collections/.../cards/<entity-id>_<slug>.yaml- collections/.../dashboards/<entity-id>_<slug>.yaml-- Directory names may differ from entity names (e.g. `analytics_data_warehouse`- for \"Analytics Data Warehouse\", or `orders` for \"ORDERS\" in H2).-- ## Lazy resolution cache-- The `SerdesSource` maintains a single lazily-populated cache atom that maps- real entity names (from YAML) to file locations:-- {db-name ; real name, e.g. \"Analytics Data Warehouse\"- {schema ; real name or nil for schema-less- ::not-indexed ; schema exists but tables not yet walked- | {table-name ; real name from YAML- {:table-file \"path/to/table.yaml\" ; path string to the table YAML- :fields ::not-indexed ; fields not yet walked for this table- | {field-name \"path\" ...}}}}} ; real name → field YAML path-- Two levels of laziness:- 1. Schema → tables: realized when any table in that schema is resolved- (via `resolve-table`, `resolve-field`, `fields-for-table`) or when- `tables-for-database` is called. Walks the schema's `tables/` directory- and reads each table's `name:` from YAML via fast regex.- 2. Table → fields: realized when `resolve-field` or `fields-for-table` is- called for that specific table. Walks the table's `fields/` directory- and reads each field's `name:` via fast regex. Other tables in the same- schema are unaffected.-- The `db-name->dir` map (built at startup from ~40 database YAMLs) handles the- database name → directory name translation. Schema and table name → directory- name translation is handled by case-insensitive directory matching during- schema indexing."- (:require- [clojure.java.io :as io]- [clojure.string :as str]- [metabase-enterprise.checker.source :as source]- [metabase.util.yaml :as yaml]- [potemkin.types :as p.types])- (:import- (java.io File)- (java.util Locale)))--(set! *warn-on-reflection* true)--(defn load-yaml- "Load and parse a YAML file."- [path]- (yaml/parse-string (slurp path)))--(defn- db-dir- "Resolve a real database name to its directory on disk."- ^File [^File databases-dir db-name->dir db-name]- (let [dir-name (get db-name->dir db-name db-name)]- (io/file databases-dir dir-name)))--;;; ===========================================================================-;;; MetadataSource Implementation-;;;-;;; Returns raw YAML data - the checker handles conversion to lib format.-;;; This keeps format knowledge here, lib knowledge in checker.-;;; ===========================================================================--(defn- resolve-schema-name- "Resolve a real schema name to the key used in the model.-- Schema names are the one entity type with no canonical YAML file — the directory- name is the filesystem key, and the real name comes from the `schema:` field in- table YAMLs. These may differ (e.g. dir=`public`, real=`PUBLIC` for H2).-- Strategy:- 1. Exact match in the model's :schemas- 2. Case-insensitive match (handles H2-style uppercasing)- 3. nil for schema-less databases-- To change schema resolution in the future, modify this function."- [model db-name schema]- (let [schemas (:schemas (get model db-name))- lower (fn [s] (.toLowerCase (str s) Locale/US))]- (cond- (contains? schemas schema) schema- schema (let [lowered (lower schema)]- (some (fn [k] (when (and (string? k) (= (lower k) lowered)) k))- (keys schemas)))- :else (when (contains? schemas nil) nil))))--(defn quick-extract- "Extract a field from YAML using regex (fast) with full parse fallback."- [file-path field-name pattern]- (try- (let [content (slurp file-path)]- (if-let [[_ value] (re-find pattern content)]- (str/trim value)- (get (yaml/parse-string content) (keyword field-name))))- (catch Exception _ nil)))--(defn- read-yaml-schema- "Read the `schema:` field from a YAML file using regex (fast).- Returns nil if not found or if the value is `null`."- [^File yaml-file]- (when (.exists yaml-file)- (let [v (quick-extract (.getPath yaml-file) "schema" #"(?m)^schema:\s*(\S+)")]- (when (and v (not= v "null"))- v))))--(defn- read-yaml-name- "Read the `name:` field from a YAML file using regex (fast).- Falls back to `fallback` if the file doesn't exist or can't be read."- [^File yaml-file ^String fallback]- (if (.exists yaml-file)- (or (quick-extract (.getPath yaml-file) "name" #"(?m)^name:\s*(.+)")- fallback)- fallback))--(defn- table-real-name- "Read the real table name from its YAML file via regex. Falls back to dir name."- [^File table-dir]- (let [dir-name (.getName table-dir)- ^File yaml (io/file table-dir (str dir-name ".yaml"))]- (read-yaml-name yaml dir-name)))--(defn- index-schema-tables- "Walk a single schema's tables directory and return:- {:real-schema-name string-or-nil ; the real schema name from YAML (may differ from dir name)- :tables {real-table-name {:table-file \"path\", :fields ::not-indexed} ...}}- Reads real table names from YAMLs via fast regex.- Fields are NOT indexed here — they are indexed lazily per-table."- [^File tables-dir]- (when (.isDirectory tables-dir)- (let [real-schema (volatile! nil)- tables (into {}- (keep (fn [^File td]- (when (.isDirectory td)- (let [table-name (table-real-name td)- dir-name (.getName td)- ^File yaml (io/file td (str dir-name ".yaml"))]- (when (.exists yaml)- (when-not @real-schema- (vreset! real-schema (read-yaml-schema yaml)))- [table-name {:table-file (.getPath yaml) :fields ::not-indexed}])))))- (.listFiles tables-dir))]- {:real-schema-name @real-schema- :tables tables})))--(defn- ensure-schema-indexed!- "Ensure a schema's tables are indexed in the model. No-op if already done.-- Resolves the requested schema name to a model key (possibly case-insensitive),- walks the table directories, reads real names from YAMLs, and stores the results.- If the real schema name (from the table YAML `schema:` field) differs from the- directory name, the model entry is re-keyed under the real name."- [^File databases-dir db-name->dir db-name schema model]- (let [model-key (resolve-schema-name @model db-name schema)]- (when (= ::not-indexed (get-in @model [db-name :schemas model-key]))- (let [^File db (db-dir databases-dir db-name->dir db-name)- ^File tables-dir (if model-key- (io/file db "schemas" model-key "tables")- (io/file db "tables"))- {:keys [real-schema-name tables]} (index-schema-tables tables-dir)- tables (or tables {})- ;; The real schema name (from YAML) is the canonical key.- ;; Falls back to model-key (dir name) or the requested name.- final-key (or real-schema-name model-key schema)]- (swap! model (fn [m]- (cond-> m- ;; Remove old key if re-keying- (and model-key (not= model-key final-key))- (update-in [db-name :schemas] dissoc model-key)- ;; Store under canonical name- true- (assoc-in [db-name :schemas final-key] tables))))))))--(defn- field-real-name- "Read the real field name from its YAML file via regex. Falls back to filename without extension."- [^File field-yaml]- (let [fname (.getName field-yaml)- fallback (str/replace fname #"\.yaml$" "")]- (read-yaml-name field-yaml fallback)))--(defn- index-table-fields- "Walk a table's fields/ directory and return a map of:- {real-field-name File ...}- Reads real field names from YAMLs via fast regex.- Skips FieldValues and FieldUserSettings files."- [table-file-path]- (let [^File table-dir (.getParentFile (io/file table-file-path))- ^File fields-dir (io/file table-dir "fields")]- (if (.isDirectory fields-dir)- (into {}- (keep (fn [^File f]- (let [fname (.getName f)]- (when (and (.isFile f)- (str/ends-with? fname ".yaml")- (not (str/includes? fname "___")))- [(field-real-name f) (.getPath f)]))))- (.listFiles fields-dir))- {})))--(defn- ensure-fields-indexed!- "Ensure a table's fields are indexed in the model. No-op if already done."- [^File databases-dir db-name->dir db-name schema table-name model]- (ensure-schema-indexed! databases-dir db-name->dir db-name schema model)- (when (= ::not-indexed (get-in @model [db-name :schemas schema table-name :fields]))- (when-let [table-file (get-in @model [db-name :schemas schema table-name :table-file])]- (let [field-map (index-table-fields table-file)]- (swap! model assoc-in [db-name :schemas schema table-name :fields] field-map)))))--(defn- index-schema-dir- "Walk a databases directory and index databases, segments, and measures.- Reads ~40 database YAMLs for real names, plus a handful of segment/measure YAMLs- for entity_ids. Tables and fields are resolved on demand from the filesystem.-- When `include-segments-measures?` is true, also walks table directories- to find segment and measure YAMLs (for export dirs). Set to false for- schema-only dirs to avoid walking thousands of table directories.-- Returns {:entries [...]- :db-name->dir {real-db-name dir-name}}."- [schema-dir]- (let [schema-dir (io/file schema-dir)- db-name->dir (volatile! {})- entries (volatile! [])]- (doseq [^File db-dir (.listFiles schema-dir)- :when (.isDirectory db-dir)]- (let [dir-name (.getName db-dir)- db-yaml (io/file db-dir (str dir-name ".yaml"))- db-name (read-yaml-name db-yaml dir-name)]- (vswap! db-name->dir assoc db-name dir-name)- (vswap! entries conj {:kind :database :ref db-name- :file (if (.exists db-yaml) (.getPath db-yaml) (.getPath db-dir))})))- {:entries @entries- :db-name->dir @db-name->dir}))--(defn build-database-dir-index- "Build index of databases from a schema directory.- Tables and fields are resolved on demand.-- Returns `{:index {:database {name file-path}} :db-name->dir {real-name dir-name}}`."- [databases-dir]- (let [{:keys [entries db-name->dir]} (index-schema-dir databases-dir)]- {:index (reduce (fn [idx {:keys [kind ref file]}]- (assoc-in idx [kind ref] file))- {}- entries)- :db-name->dir db-name->dir}))--(p.types/defprotocol+ ISchemaModelDebug- "Non exported protocol to get the schema of the SerdesSource. Useful for debugging and asserting on the schema model- built up during serdes."- (schema-model [_] "Deref and return the schema model. See [[init-schema-model]] for details of this structure."))--(deftype SerdesSchemaSource [databases-dir db-name->dir- ;; See init-schema-model for shape documentation- schema-model]- source/SchemaSource- (resolve-database [_ db-name]- (when-let [file (get-in @schema-model [db-name :db-file])]- (load-yaml file)))-- (resolve-table [_ [db schema table-name :as _table-path]]- (ensure-schema-indexed! databases-dir db-name->dir db schema schema-model)- (when-let [path (get-in @schema-model [db :schemas schema table-name :table-file])]- (load-yaml path)))-- (resolve-field [_ [db schema table-name field :as _field-path]]- (ensure-fields-indexed! databases-dir db-name->dir db schema table-name schema-model)- (when-let [path (get-in @schema-model [db :schemas schema table-name :fields field])]- (load-yaml path)))-- (fields-for-table [_ [db schema table-name :as _table-path]]- (ensure-fields-indexed! databases-dir db-name->dir db schema table-name schema-model)- (when-let [entry (get-in @schema-model [db :schemas schema table-name])]- (let [fields (:fields entry)]- (when (map? fields)- (into #{} (map (fn [field-name] [db schema table-name field-name])) (keys fields))))))-- (all-field-paths [_]- (doseq [[db-name {:keys [schemas]}] @schema-model- [schema tables] schemas- :when (= tables ::not-indexed)]- (ensure-schema-indexed! databases-dir db-name->dir db-name schema schema-model))- (doseq [[db-name {:keys [schemas]}] @schema-model- [schema tables] schemas- :when (map? tables)- [table-name entry] tables- :when (= (:fields entry) ::not-indexed)]- (ensure-fields-indexed! databases-dir db-name->dir db-name schema table-name schema-model))- (into #{}- (for [[db {:keys [schemas]}] @schema-model- [schema tables] schemas- :when (map? tables)- [table-name {:keys [fields]}] tables- :when (map? fields)- field-name (keys fields)]- [db schema table-name field-name])))-- (all-database-names [_]- (keys @schema-model))-- (all-table-paths [_]- (doseq [[db-name {:keys [schemas]}] @schema-model- [schema _] schemas]- (ensure-schema-indexed! databases-dir db-name->dir db-name schema schema-model))- (into []- (for [[db {:keys [schemas]}] @schema-model- [schema tables] schemas- :when (map? tables)- table-name (keys tables)]- [db schema table-name])))-- (tables-for-database [_ db-name]- (doseq [[schema _] (:schemas (get @schema-model db-name))]- (ensure-schema-indexed! databases-dir db-name->dir db-name schema schema-model))- (into []- (for [[schema tables] (:schemas (get @schema-model db-name))- :when (map? tables)- table-name (keys tables)]- [db-name schema table-name])))-- ISchemaModelDebug- (schema-model [_] (deref schema-model)))--(defn- list-schemas- "List schema directory names for a database. Returns seq of strings, or [nil] for schema-less."- [^File databases-dir db-name->dir db-name]- (let [^File db (db-dir databases-dir db-name->dir db-name)- ^File schemas-dir (io/file db "schemas")- ^File tables-dir (io/file db "tables")]- (cond-> []- (.isDirectory schemas-dir)- (into (map #(.getName ^File %))- (filter #(.isDirectory ^File %) (.listFiles schemas-dir)))- (.isDirectory tables-dir)- (conj nil))))--(defn- init-schema-model- "Build the initial schema model with db-file paths and ::not-indexed for each schema.-- Schema model shape:- {db-name {:db-file \"path/to/db.yaml\"- :schemas {schema ::not-indexed- | {table {:table-file \"path\" :fields ::not-indexed | {field \"path\"}}}}}}-- Two levels of laziness:- 1. Schema → tables: populated when the schema is first accessed- 2. Table → fields: populated when that table's fields are first accessed"- [^File databases-dir db-name->dir db-file-index]- (atom- (into {}- (for [[db-name _dir-name] db-name->dir]- [db-name {:db-file (get db-file-index db-name)- :schemas (into {}- (for [schema (list-schemas databases-dir db-name->dir db-name)]- [schema ::not-indexed]))}]))))--(defn make-database-source- "Create a MetadataSource for a directory that IS the databases directory.- The directory should contain database subdirectories directly (e.g., `Sample Database/`).- This source only resolves databases, tables, and fields — not cards."- [databases-dir]- (let [{:keys [index db-name->dir]} (build-database-dir-index databases-dir)- databases-dir (io/file databases-dir)- db-files (:database index)]- (->SerdesSchemaSource databases-dir db-name->dir- (init-schema-model databases-dir db-name->dir db-files))))diff --git a/enterprise/backend/src/metabase_enterprise/checker/provider.clj b/enterprise/backend/src/metabase_enterprise/checker/provider.cljdeleted file mode 100644index 11fc71263fb..00000000000--- a/enterprise/backend/src/metabase_enterprise/checker/provider.clj+++ /dev/null@@ -1,466 +0,0 @@-(ns metabase-enterprise.checker.provider- "Adapts a checker store into a lib MetadataProvider.-- The store (checker.store) knows what entities exist and can load them.- This namespace converts that raw data into the lib metadata format that- `lib/query` and `deps.analysis/check-entity` expect.-- Responsibilities:- - Convert raw YAML entity maps → lib metadata maps (keywordize types,- resolve FK targets, normalize MBQL operators, resolve portable refs)- - Implement `MetadataProvider` by delegating to the store- - Assign sentinel IDs for unresolved refs so queries can still be- constructed and `lib/find-bad-refs` reports issues with context-- Public API:- - `(make-provider store)` — wrap a store in a MetadataProvider- - `(card-metadata store data)` — convert a card, collecting resolution failures- - `(transform-metadata store data)` — convert a transform, collecting resolution failures"- (:require- [metabase-enterprise.checker.store :as store]- [metabase.lib.core :as lib]- [metabase.lib.metadata.protocols :as lib.metadata.protocols]- [metabase.lib.schema.metadata :as lib.schema.metadata]- [metabase.models.serialization.resolve :as resolve]- [potemkin.types :as p.types]))--(set! *warn-on-reflection* true)--;;; ===========================================================================-;;; Settable database protocol-;;; ===========================================================================--(p.types/defprotocol+ ISettableDatabase- "Protocol for providers that can switch which database they report."- (set-database! [this db-name]- "Set the database name (string) this provider should return from `(database)`.")- (clear-database! [this]- "Clear the current database. `(database)` will return nil until `set-database!` is called again."))--;;; ===========================================================================-;;; Reference resolution-;;;-;;; Resolution functions take the store and a `!failures` atom (or nil).-;;; When a ref can't be resolved, a failure map is conj'd onto the atom.-;;; ===========================================================================--(defn- resolve-ref- "Try to resolve `ref` of `kind`. Returns integer ID or nil.- On failure, appends failure-info to `!failures`.- When `sentinel?` is true, assigns a sentinel ID for unresolved refs."- [store kind ref !failures failure-info & {:keys [sentinel?]}]- (when ref- (or (store/ref->id store kind ref)- (let [load-fn (case kind- :database store/load-database!- :table store/load-table!- :field store/load-field!- :card store/load-card!- :snippet store/load-snippet!- :transform store/load-transform!- :segment store/load-segment!- :dashboard store/load-dashboard!- :collection store/load-collection!- :document store/load-document!- :measure store/load-measure!)- found? (some? (load-fn store ref))]- (if found?- (store/get-or-assign! store kind ref)- (do (when !failures (swap! !failures conj failure-info))- (when sentinel?- (store/get-or-assign! store kind ref))))))))--(defn- resolve-field-path- ([store !failures field-path]- (resolve-ref store :field field-path !failures {:type :field :path field-path}))- ([store !failures field-path sentinel?]- (resolve-ref store :field field-path !failures {:type :field :path field-path} :sentinel? sentinel?)))--(defn- resolve-table-path- ([store !failures table-path]- (resolve-ref store :table table-path !failures {:type :table :path table-path}))- ([store !failures table-path sentinel?]- (resolve-ref store :table table-path !failures {:type :table :path table-path} :sentinel? sentinel?)))--(defn- resolve-db-name [store !failures db-name]- (resolve-ref store :database db-name !failures {:type :database :name db-name}))--(defn- resolve-card-entity-id [store !failures entity-id]- (resolve-ref store :card entity-id !failures {:type :card :entity-id entity-id}))--;;; ----------------------------------------------------------------------------;;; SerdesImportResolver — bridges store-based resolution into import-mbql-;;; -----------------------------------------------------------------------------(defn- make-import-resolver- [store !failures & {:keys [sentinel?]}]- (reify resolve/SerdesImportResolver- (import-fk [_ eid model]- #_{:clj-kondo/ignore [:case-symbol-test]}- (case model- Card (resolve-card-entity-id store !failures eid)- Segment (resolve-ref store :segment eid !failures {:type :segment :entity-id eid})- Measure (resolve-ref store :measure eid !failures {:type :measure :entity-id eid})- NativeQuerySnippet (resolve-ref store :snippet eid !failures {:type :snippet :entity-id eid})- (do (when !failures- (swap! !failures conj {:type :unknown :model model :entity-id eid}))- nil)))- (import-fk-keyed [_ portable model field]- (case [model field]- [:model/Database :name] (resolve-db-name store !failures portable)- (do (when !failures- (swap! !failures conj {:type :keyed-lookup :model model :field field :value portable}))- nil)))- (import-user [_ _email] nil)- (import-table-fk [_ path] (resolve-table-path store !failures path sentinel?))- (import-field-fk [_ path] (resolve-field-path store !failures path sentinel?))))--;;; ===========================================================================-;;; Data conversion — raw YAML entity maps → lib metadata maps-;;; ===========================================================================--(defn- ->database-metadata [data]- {:lib/type :metadata/database- :id (:id data)- :name (:name data)- :engine (keyword (:engine data))- :dbms-version (:dbms_version data)- :features #{:metadata/key-constraints :nested-queries :expressions :native-parameters- :basic-aggregations :standard-deviation-aggregations- :expression-aggregations :left-join :right-join :inner-join :full-join}- :settings (:settings data)})--(defn- ->table-metadata [data]- {:lib/type :metadata/table- :id (:id data)- :name (:name data)- :display-name (:display_name data)- :schema (:schema data)- :db-id (:db_id data)- :active (if (contains? data :active) (:active data) true)- :visibility-type (some-> (:visibility_type data) keyword)})--(defn- ->field-metadata- [resolver data]- (cond-> {:lib/type :metadata/column- :id (:id data)- :table-id (:table_id data)- :name (:name data)- :display-name (:display_name data)- :base-type (keyword (:base_type data))- :effective-type (or (some-> (:effective_type data) keyword)- (keyword (:base_type data)))- :semantic-type (some-> (:semantic_type data) keyword)- :database-type (:database_type data)- :active (if (contains? data :active) (:active data) true)- :visibility-type (some-> (:visibility_type data) keyword)- :position (:position data)}- (and resolver (vector? (:fk_target_field_id data)))- (as-> m (if-let [fk-id (resolve/import-field-fk resolver (:fk_target_field_id data))]- (assoc m :fk-target-field-id fk-id)- m))))--(defn- convert-result-metadata-column- "Resolve portable refs in a result_metadata column map.- Portable refs appear as path vectors in :id, :table_id, :fk_target_field_id,- and as MBQL field refs in :field_ref."- [resolver col]- (let [resolve-or-drop (fn [col k resolve-fn]- (let [v (get col k)]- (if-not (vector? v)- col- (if-let [id (resolve-fn resolver v)]- (assoc col k id)- (dissoc col k)))))]- (-> col- (resolve-or-drop :id resolve/import-field-fk)- (resolve-or-drop :table_id resolve/import-table-fk)- (resolve-or-drop :fk_target_field_id resolve/import-field-fk)- (cond-> (:field_ref col)- (update :field_ref (fn [ref]- (let [resolved (resolve/import-mbql resolver ref)]- (if (and (vector? resolved) (= :field (first resolved)) (nil? (second resolved)))- ::drop- resolved)))))- (as-> c (if (= ::drop (:field_ref c)) (dissoc c :field_ref) c)))))--(defn- convert-dataset-query- "Convert a dataset query from serdes format, resolving paths to IDs.-- Pipeline:- 1. lib/normalize — convert YAML strings to keywords (operators, types, etc.)- 2. resolve/import-mbql — resolve portable refs (path vectors) to integer IDs- Uses a sentinel resolver so unresolved refs get IDs for better error reporting."- [resolver query]- (when query- (try- (let [query (lib/normalize nil query)- db-name (:database query)- db-id (when (string? db-name)- (resolve/import-fk-keyed resolver db-name :model/Database :name))- query (cond-> query db-id (assoc :database db-id))- query (resolve/import-mbql resolver query)]- query)- (catch Exception _- (let [db-name (:database query)- db-id (when (string? db-name)- (resolve/import-fk-keyed resolver db-name :model/Database :name))]- (cond-> query db-id (assoc :database db-id)))))))--;;; ----------------------------------------------------------------------------;;; High-level entity conversion (public — used by semantic.clj)-;;; -----------------------------------------------------------------------------(declare compute-result-metadata)--(defn card-metadata- "Convert a cached card entity to lib metadata. Returns the metadata map with- `:provider/resolution-failures` and `:provider/missing-database` keys when resolution fails."- [store data]- (let [!failures (atom [])- ;; Sentinel resolver: unresolved refs get fake integer IDs so queries can still be constructed and- ;; lib/find-bad-refs reports issues with context. We use the same resolver for both the query and metadata- ;; (table_id, FK targets, result_metadata). If we ever need to distinguish "soft failure" (metadata)- ;; from "must-build" (query), create a second resolver with :sentinel? false and use it for the metadata- ;; resolution calls below.- resolver (make-import-resolver store !failures :sentinel? true)- table-id (when-let [t (:table_id data)]- (if (vector? t)- (resolve/import-table-fk resolver t)- t))- db-name (get-in data [:dataset_query :database])- db-id (when (string? db-name)- (resolve/import-fk-keyed resolver db-name :model/Database :name))- dataset-query (convert-dataset-query resolver (:dataset_query data))- result-metadata (if-let [cols (seq (:result_metadata data))]- (->> cols- (map (partial convert-result-metadata-column resolver))- (lib/normalize [:sequential ::lib.schema.metadata/lib-or-legacy-column]))- (when (and dataset-query db-id)- (compute-result-metadata store dataset-query)))]- (cond-> {:lib/type :metadata/card- :id (:id data)- :name (:name data)- :type (some-> (:type data) keyword)- :dataset-query dataset-query- :result-metadata result-metadata- :archived (:archived data)}- db-id (assoc :database-id db-id)- (not db-id) (assoc :provider/missing-database db-name)- table-id (assoc :table-id table-id)- (seq @!failures) (assoc :provider/resolution-failures (vec (distinct @!failures))))))--(defn transform-metadata- "Convert a cached transform entity to lib metadata with its source query resolved."- [store data]- (let [!failures (atom [])- resolver (make-import-resolver store !failures :sentinel? true)- query (convert-dataset-query resolver (get-in data [:source :query]))]- (cond-> {:lib/type :metadata/transform- :id (:id data)- :name (:name data)- :source (assoc (:source data) :query query)}- (seq @!failures) (assoc :provider/resolution-failures (vec (distinct @!failures))))))--(defn- ->segment-metadata- [store data]- (let [!failures (atom [])- resolver (make-import-resolver store !failures :sentinel? true)- definition (convert-dataset-query resolver (:definition data))]- (cond-> {:lib/type :metadata/segment- :id (:id data)- :name (:name data)- :definition definition}- (seq @!failures) (assoc :provider/resolution-failures (vec (distinct @!failures))))))--;;; ===========================================================================-;;; MetadataProvider-;;; ===========================================================================--(deftype SourceMetadataProvider- ;; The store knows what entities exist, loads raw YAML data, assigns integer IDs, and caches.- ;; The provider adapts the store into the MetadataProvider interface that lib/query expects,- ;; converting raw YAML maps to lib metadata format along the way.- ;;- ;; `current-db` is an atom holding the database name (string) to return from `(database)`.- ;; Must be set via `set-database!` before each entity check and unset afterward.- [store current-db]- ISettableDatabase- (set-database! [_this db-name]- (reset! current-db db-name))- (clear-database! [_this]- (reset! current-db nil))-- lib.metadata.protocols/MetadataProvider- (database [_this]- (let [db-name @current-db]- (when (nil? db-name)- (throw (ex-info "No current database set on provider. Call set-database! before querying." {})))- (when-let [data (store/load-database! store db-name)]- (->database-metadata data))))-- (metadatas [_this {:keys [lib/type id table-ids name]}]- (case type- :metadata/table- (vec- (if id- (for [tid id- :let [path (store/id->ref store :table tid)]- :when path- :let [data (store/load-table! store path)]- :when data]- (->table-metadata data))- ;; Only return tables for the current database — uses lightweight- ;; path-derived metadata, no YAML parsing- (let [db-name @current-db]- (for [path (store/tables-for-database store db-name)- :let [data (store/ensure-table-id! store path)]- :when data]- (->table-metadata data)))))-- :metadata/column- (vec- (cond- id- (for [fid id- :let [path (store/id->ref store :field fid)]- :when path- :let [data (store/load-field! store path)]- :when data]- (->field-metadata (make-import-resolver store nil) data))-- table-ids- (for [table-id table-ids- :let [table-path (store/id->ref store :table table-id)]- :when table-path- fpath (store/fields-for-table store table-path)- :let [data (store/load-field! store fpath)]- :when data]- (->field-metadata (make-import-resolver store nil) data))-- :else- (for [path (store/all-field-paths store)- :let [data (store/load-field! store path)]- :when data]- (->field-metadata (make-import-resolver store nil) data))))-- :metadata/card- (vec- (if id- (for [cid id- :let [eid (store/id->ref store :card cid)]- :when eid- :let [data (store/load-card! store eid)]- :when data]- (card-metadata store data))- (for [eid (store/all-card-ids store)- :let [data (store/load-card! store eid)]- :when data]- (card-metadata store data))))-- :metadata/native-query-snippet- (vec- (cond- id- (for [sid id- :let [eid (store/id->ref store :snippet sid)]- :when eid- :let [data (store/load-snippet! store eid)]- :when data]- {:lib/type :metadata/native-query-snippet- :id (:id data)- :name (:name data)- :content (:content data)})-- name- (for [eid (store/all-refs store :snippet)- :let [data (store/load-snippet! store eid)]- :when data- :when (contains? name (:name data))]- {:lib/type :metadata/native-query-snippet- :id (:id data)- :name (:name data)- :content (:content data)})-- :else []))-- :metadata/transform- (vec- (if id- (for [tid id- :let [eid (store/id->ref store :transform tid)]- :when eid- :let [data (store/load-transform! store eid)]- :when data]- (transform-metadata store data))- (for [eid (store/all-refs store :transform)- :let [data (store/load-transform! store eid)]- :when data]- (transform-metadata store data))))-- :metadata/segment- (vec- (if id- (for [sid id- :let [eid (store/id->ref store :segment sid)]- :when eid- :let [data (store/load-segment! store eid)]- :when data]- (->segment-metadata store data))- (for [eid (store/all-refs store :segment)- :let [data (store/load-segment! store eid)]- :when data]- (->segment-metadata store data))))-- nil))-- (setting [_this _key] nil)-- lib.metadata.protocols/CachedMetadataProvider- (cached-metadatas [this type ids]- (lib.metadata.protocols/metadatas this {:lib/type type :id (set ids)}))- (store-metadata! [_this _obj] nil)- (cached-value [_this _k not-found] not-found)- (cache-value! [_this _k _v] nil)- (has-cache? [_this] true)- (clear-cache! [_this] nil))--(declare make-provider)--(defn- compute-result-metadata- [store dataset-query]- (try- (let [provider (make-provider store)- db-id (:database dataset-query)- db-name (when (integer? db-id) (store/id->ref store :database db-id))]- (when db-name- (set-database! provider db-name))- (let [query (lib/query provider dataset-query)]- (mapv #(select-keys % [:name :base-type :effective-type :semantic-type :display-name])- (lib/returned-columns query))))- (catch Exception _ nil)))--;;; ===========================================================================-;;; Public API-;;; ===========================================================================--(defn make-provider- "Create a MetadataProvider backed by a store.- The store holds sources, index, ID registry, and entity caches —- see `checker.store/make-store`."- [store]- (->SourceMetadataProvider store (atom nil)))--(defn database-name-for-entity- "Determine the database name for an entity from its raw YAML data.- Cards have it in dataset_query.database, transforms in source.query.database."- [store kind data]- (let [db-ref (case kind- :card (get-in data [:dataset_query :database])- :transform (or (get-in data [:source :query :database])- (:source_database_id data))- nil)]- (cond- (string? db-ref) db-ref- (integer? db-ref) (store/id->ref store :database db-ref)- :else nil)))diff --git a/enterprise/backend/src/metabase_enterprise/checker/semantic.clj b/enterprise/backend/src/metabase_enterprise/checker/semantic.cljdeleted file mode 100644index 9842d8f000f..00000000000--- a/enterprise/backend/src/metabase_enterprise/checker/semantic.clj+++ /dev/null@@ -1,871 +0,0 @@-(ns metabase-enterprise.checker.semantic- "Entity validation — the checker's main entry point.-- Creates a store (checker.store) from sources, wraps it in a MetadataProvider- (checker.provider), then validates every entity:- - Query validation via deps.analysis/check-entity (MBQL + native SQL)- - Structural checks (collection_id, dashboard layout, document links)-- Public API:- - `(check export-dir schema-dir)` — check all entities, returns results map- - `(setup export-dir schema-dir)` + `(check-one ctx entity-id)` — REPL workflow"- (:require- [clojure.string :as str]- [metabase-enterprise.checker.format.concise-schema :as concise-schema]- [metabase-enterprise.checker.format.serdes-assets :as serdes-assets]- [metabase-enterprise.checker.format.serdes-schema :as serdes.schema]- [metabase-enterprise.checker.provider :as provider]- [metabase-enterprise.checker.store :as store]- [metabase-enterprise.dependencies.analysis :as deps.analysis]- [metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata]- [metabase.query-processor.interface :as qp.i]- ;; sql-tools.init registers multimethod implementations needed by deps.native-validation- [metabase.sql-tools.init]- [metabase.util.malli.fn :as mu.fn]))--(set! *warn-on-reflection* true)--;;; ===========================================================================-;;; Reference resolution — resolve portable refs to IDs, collecting failures-;;;-;;; Resolution functions take the store and a `!failures` atom (or nil).-;;; When a ref can't be resolved, a failure map is conj'd onto the atom-;;; for the caller to inspect. The failure maps look like:-;;; {:type :field :path ["DB" "schema" "table" "field"]}-;;; {:type :table :path ["DB" "schema" "table"]}-;;; {:type :database :name "DB"}-;;; ===========================================================================-;;; Entity loading by kind-;;; ===========================================================================--(def ^:private kind->load-fn- {:dashboard store/load-dashboard!- :collection store/load-collection!- :document store/load-document!- :transform store/load-transform!- :segment store/load-segment!- :measure store/load-measure!- :snippet store/load-snippet!})--;;; ===========================================================================-;;; Common entity checks — apply to any entity type-;;;-;;; These validations run on every entity (cards, dashboards, collections,-;;; metrics, segments, etc.). Add new cross-entity checks here.-;;; ===========================================================================--(defn- validate-collection-id- "Validate that `collection_id` in `data` points to a known collection.- Returns a failure map or nil."- [store data]- (when-let [coll-id (:collection_id data)]- (let [kind (store/index-kind-of store coll-id)]- (cond- (nil? kind)- {:type :collection :entity-id coll-id- :message (str "collection_id " coll-id " not found")}-- (not= :collection kind)- {:type :collection :entity-id coll-id- :message (str "collection_id " coll-id " points to a " (name kind) ", not a collection")}))))--(defn- validate-dashboard-id- "Validate that `dashboard_id` in `data` points to a known dashboard.- Returns a failure map or nil."- [store data]- (when-let [dash-id (:dashboard_id data)]- (let [kind (store/index-kind-of store dash-id)]- (cond- (nil? kind)- {:type :dashboard :entity-id dash-id- :message (str "dashboard_id " dash-id " not found")}-- (not= :dashboard kind)- {:type :dashboard :entity-id dash-id- :message (str "dashboard_id " dash-id " points to a " (name kind) ", not a dashboard")}))))--(defn- validate-parent-id- "Validate that `parent_id` on a collection points to a known collection.- Returns a failure map or nil."- [store data]- (when-let [parent-id (:parent_id data)]- (let [kind (store/index-kind-of store parent-id)]- (cond- (nil? kind)- {:type :collection :entity-id parent-id- :message (str "parent_id " parent-id " not found")}-- (not= :collection kind)- {:type :collection :entity-id parent-id- :message (str "parent_id " parent-id " points to a " (name kind) ", not a collection")}))))--(defn- validate-document-id- "Validate that `document_id` in `data` points to a known document.- Returns a failure map or nil."- [store data]- (when-let [doc-id (:document_id data)]- (let [kind (store/index-kind-of store doc-id)]- (cond- (nil? kind)- {:type :document :entity-id doc-id- :message (str "document_id " doc-id " not found")}-- (not= :document kind)- {:type :document :entity-id doc-id- :message (str "document_id " doc-id " points to a " (name kind) ", not a document")}))))--(defn- validate-container-exclusion- "Validate that a card doesn't have both dashboard_id and document_id set.- Returns a failure map or nil."- [_store data]- (when (and (:dashboard_id data) (:document_id data))- {:type :container-conflict- :message "card has both dashboard_id and document_id set — only one container allowed"}))--(defn- validate-container-collection-match- "Validate that a card nested under a dashboard or document has the same- collection_id as its container. Returns a failure map or nil."- [store data]- (when-let [container-id (or (:dashboard_id data) (:document_id data))]- (let [container-kind (if (:dashboard_id data) :dashboard :document)- load-fn (kind->load-fn container-kind)- container-data (when load-fn (load-fn store container-id))]- (when container-data- (let [container-coll (:collection_id container-data)- card-coll (:collection_id data)]- (when (and card-coll container-coll- (not= card-coll container-coll))- {:type :container-collection-mismatch- :message (str "card collection_id " card-coll- " doesn't match its " (name container-kind)- "'s collection_id " container-coll)}))))))--(defn- check-common- "Run checks that apply to any entity (card, dashboard, metric, etc.).- Returns a vector of failure maps, or empty vector if all checks pass."- [store data]- (filterv some?- [(validate-collection-id store data)- (validate-dashboard-id store data)- (validate-document-id store data)- (validate-parent-id store data)- (validate-container-exclusion store data)- (validate-container-collection-match store data)]))--;;; ===========================================================================-;;; Dashboard-specific checks-;;;-;;; Validates dashboard internal consistency:-;;; - dashcard card_id refs point to known cards-;;; - dashcard dashboard_tab_id refs point to tabs in this dashboard-;;; - dashcard grid positions are within bounds (24-column grid)-;;; ===========================================================================--(def ^:private grid-width- "Dashboard grid width in columns."- 24)--(defn- validate-dashcard-card-ref- "Validate that a dashcard's card_id points to a known card.- Virtual cards (headings, text) have null card_id — that's fine."- [store dashcard]- (let [card-id (:card_id dashcard)]- (when (and card-id (not (store/exists? store :card card-id)))- {:type :dashcard-card-ref- :dashcard-entity-id (:entity_id dashcard)- :entity-id card-id- :message (str "dashcard " (:entity_id dashcard) " references card " card-id " which is not in the export")})))--(defn- validate-dashcard-tab-ref- "Validate that a dashcard's dashboard_tab_id references a tab in this dashboard.- dashboard_tab_id is [dashboard-entity-id, tab-entity-id]."- [tab-entity-ids dashcard]- (when-let [tab-ref (:dashboard_tab_id dashcard)]- (let [tab-eid (if (vector? tab-ref) (second tab-ref) tab-ref)]- (when (and tab-eid (not (contains? tab-entity-ids tab-eid)))- {:type :dashcard-tab-ref- :dashcard-entity-id (:entity_id dashcard)- :entity-id tab-eid- :message (str "dashcard " (:entity_id dashcard) " references tab " tab-eid " which is not in this dashboard")}))))--(defn- validate-dashcard-grid- "Validate that a dashcard's grid position is within bounds."- [dashcard]- (let [col (or (:col dashcard) 0)- row (or (:row dashcard) 0)- size-x (or (:size_x dashcard) 1)- size-y (or (:size_y dashcard) 1)- eid (:entity_id dashcard)]- (cond- (neg? col)- {:type :dashcard-grid :dashcard-entity-id eid- :message (str "dashcard " eid " has negative col: " col)}-- (> (+ col size-x) grid-width)- {:type :dashcard-grid :dashcard-entity-id eid- :message (str "dashcard " eid " extends beyond grid: col=" col " + size_x=" size-x " > " grid-width)}-- (< size-x 1)- {:type :dashcard-grid :dashcard-entity-id eid- :message (str "dashcard " eid " has invalid size_x: " size-x)}-- (< size-y 1)- {:type :dashcard-grid :dashcard-entity-id eid- :message (str "dashcard " eid " has invalid size_y: " size-y)}-- (neg? row)- {:type :dashcard-grid :dashcard-entity-id eid- :message (str "dashcard " eid " has negative row: " row)})))--(defn- check-dashboard- "Run dashboard-specific semantic checks on loaded dashboard data.- Returns a vector of failure maps."- [store data]- (let [tabs (:tabs data)- dashcards (:dashcards data)- tab-entity-ids (into #{} (keep :entity_id) tabs)]- (into []- (keep identity)- (mapcat (fn [dc]- [(validate-dashcard-card-ref store dc)- (validate-dashcard-tab-ref tab-entity-ids dc)- (validate-dashcard-grid dc)])- dashcards))))--;;; ===========================================================================-;;; Document-specific checks-;;;-;;; Documents contain ProseMirror content with embedded card references.-;;; cardEmbed nodes have attrs.id which is a serdes portable ref:-;;; [{:model "Card" :id "entity-id"}]-;;; ===========================================================================--(defn- extract-card-embeds- "Walk a ProseMirror content tree and extract card entity-ids from cardEmbed nodes."- [node]- (when (map? node)- (let [embeds (when (= "cardEmbed" (:type node))- (let [refs (get-in node [:attrs :id])]- (keep (fn [ref]- (when (and (map? ref) (= "Card" (:model ref)))- (:id ref)))- (if (sequential? refs) refs [refs]))))- child-embeds (mapcat extract-card-embeds (:content node))]- (concat embeds child-embeds))))--(defn- extract-entity-links- "Walk a ProseMirror content tree and extract entity references from link hrefs.- Links like /dashboard/ABC or /question/ABC reference entities by entity_id."- [node]- (when (map? node)- (let [;; Check marks on text nodes for links- mark-links (when-let [marks (:marks node)]- (keep (fn [mark]- (when (= "link" (:type mark))- (when-let [href (get-in mark [:attrs :href])]- (when-let [[_ kind eid] (re-matches #"/(dashboard|question|card|collection)/(.+)" (str href))]- {:kind (case kind- "dashboard" :dashboard- "question" :card- "card" :card- "collection" :collection)- :entity-id eid}))))- marks))- child-links (mapcat extract-entity-links (:content node))]- (concat mark-links child-links))))--(defn- check-document- "Run document-specific semantic checks. Validates embedded card refs and entity links."- [store data]- (let [doc (:document data)- card-eids (extract-card-embeds doc)- links (extract-entity-links doc)]- (into []- cat- [;; Check card embeds- (keep (fn [card-eid]- (when-not (store/exists? store :card card-eid)- {:type :document-card-ref- :entity-id card-eid- :message (str "document embeds card " card-eid " which is not in the export")}))- card-eids)- ;; Check entity links in href attributes- (keep (fn [{:keys [kind entity-id]}]- (when-not (store/exists? store kind entity-id)- {:type :document-link- :entity-id entity-id- :message (str "document links to " (name kind) " " entity-id " which is not in the export")}))- links)])))--;;; ===========================================================================-;;; deps.analysis helpers-;;; ===========================================================================--(defn- partition-errors- "Split a set of check-entity errors into :bad-refs (MBQL issues) and- :native-errors (SQL validation issues like :missing-column, :syntax-error)."- [errors]- (let [{native true, mbql false} (group-by #(contains? #{:missing-column :syntax-error :missing-table-alias} (:type %))- errors)]- {:bad-refs (vec mbql)- :native-errors (when (seq native) (set native))}))--(defn- humanize-source-entity- "Replace synthetic :source-entity-id integers with portable refs (entity-ids- or path strings) so error output is meaningful."- [store error]- (if-let [src-id (:source-entity-id error)]- (let [kind (:source-entity-type error)- ref (case kind- :table (some->> (store/id->ref store :table src-id) (str/join "."))- :card (or (store/id->ref store :card src-id) src-id)- src-id)]- (assoc error :source-entity-id ref))- error))--(defn- humanize-errors- "Translate all synthetic IDs in a set of errors back to portable refs."- [store errors]- (when errors- (into (empty errors) (map #(humanize-source-entity store %)) errors)))--;;; ===========================================================================-;;; Transform-specific checks-;;;-;;; Transforms have a source query (native or MBQL) and a target table.-;;; We validate the database ref and the source query via deps.analysis.-;;; ===========================================================================--(defn- check-transform- "Run transform-specific semantic checks via deps.analysis/check-entity.- Loads the transform from the store (which caches and assigns IDs), then- delegates query validation to deps.analysis.- Returns a result map with :unresolved, :native-errors, :bad-refs, :error, :sql keys."- [store provider entity-id]- (let [data (store/load-transform! store entity-id)]- (if (not= "query" (get-in data [:source :type]))- ;; Non-query transforms (e.g. python) — no query to validate- {:unresolved []}- ;; Query-based transforms — validate via deps.analysis- (let [db-name (or (get-in data [:source :query :database])- (:source_database_id data))- unresolved (atom [])- transform-id (:id data)- sql (get-in data [:source :query :stages 0 :native])]- ;; Check database ref- (when (and db-name (not (store/exists? store :database db-name)))- (swap! unresolved conj {:type :database :name db-name- :message (str "transform references database " db-name " which is not in the schema")}))- ;; Resolution failures from converting transform metadata- (let [transform-meta (provider/transform-metadata store data)]- (swap! unresolved into (or (:provider/resolution-failures transform-meta) []))- ;; Validate via deps.analysis- (let [errors (try- (deps.analysis/check-entity provider :transform transform-id)- (catch Exception e- #{{:type :validation-exception-error :message (.getMessage e)}}))- {:keys [bad-refs native-errors]} (partition-errors errors)- native-errors (humanize-errors store native-errors)- bad-refs (mapv #(humanize-source-entity store %) bad-refs)]- (cond-> {:unresolved @unresolved}- (seq bad-refs) (assoc :bad-refs bad-refs)- (seq native-errors) (assoc :native-errors native-errors)- sql (assoc :sql sql))))))))--;;; ===========================================================================-;;; Non-card entity checking — load from files, run common + type-specific checks-;;; ===========================================================================--(defn- id->path- "Convert an integer ID back to a human-readable path string for error messages."- [store id id-type]- (case id-type- :table (some->> (store/id->ref store :table id) (str/join "."))- :field (some->> (store/id->ref store :field id) (str/join "."))- :card (when-let [eid (store/id->ref store :card id)]- (or (:name (store/cached-entity store :card eid)) eid))- :database (store/id->ref store :database id)- nil))--(declare extract-refs-from-card)--(defn- merge-refs- "Merge multiple ref maps, concatenating and deduplicating each key."- [& ref-maps]- (reduce (fn [acc m]- (reduce-kv (fn [a k v]- (if (seq v)- (update a k (fn [existing]- (vec (distinct (into (or existing []) v)))))- a))- acc m))- {} ref-maps))--(declare extract-entity-refs-from-raw-data)--(defn- extract-refs-from-query- ([store query] (extract-refs-from-query store query nil #{}))- ([store query provider visited]- (let [table-ids (lib/all-source-table-ids query)- card-ids (lib/all-source-card-ids query)- cols (try (lib/returned-columns query) (catch Exception _ nil))- tables (mapv #(id->path store % :table) table-ids)- fields (when cols (->> cols (keep :id) (mapv #(id->path store % :field))))- cards (mapv #(id->path store % :card) card-ids)- transitive (when provider- (for [cid card-ids :when (not (visited cid))]- (extract-refs-from-card store provider cid (conj visited cid))))- direct {:tables (vec (remove nil? tables))- :fields (vec (remove nil? fields))- :source-cards (vec (remove nil? cards))}]- (apply merge-refs direct transitive))))--(defn- extract-refs-from-card [store provider card-id visited]- (try- (when-let [card (lib.metadata/card provider card-id)]- (let [;; Get the raw card data from the store for entity ref extraction- eid (store/id->ref store :card card-id)- raw-data (when eid (store/cached-entity store :card eid))- entity-refs (when raw-data (extract-entity-refs-from-raw-data raw-data))- query-refs (when-let [dq (:dataset-query card)]- (extract-refs-from-query store (lib/query provider dq) provider visited))]- (merge-refs (or query-refs {}) entity-refs)))- (catch Exception _ {})))--(defn- extract-entity-refs-from-raw-data- "Extract snippet, measure, metric, and segment references from raw card data.- These are extracted from the YAML structure before resolution, since the- resolved lib query normalizes them away."- [data]- (let [stages (get-in data [:dataset_query :stages])]- (reduce- (fn [acc stage]- (let [template-tags (:template-tags stage)- ;; TODO (Cam 2026-07-07) MEGA ICK - we have to copy the usual template tag normalization stuff (map =>- ;; list) done IN LIB here because this code is DOING ⚠⚠⚠ RAW MBQL INTROSPECTION OUTSIDE OF LIB ⚠⚠⚠ which- ;; means it has to handle both old and new shapes instead of letting Lib take care of that for us... who- ;; know what other shape variations this code doesn't handle correctly? (See GHY-4080)- template-tags (if (map? template-tags)- (into []- (map (fn [[tag-name tag]]- (assoc tag :name tag-name)))- template-tags)- template-tags)- ;; Snippets from template-tags- snippets (->> template-tags- (filter #(= "snippet" (get % :type)))- (keep :snippet-name))- ;; Measures and metrics from aggregation clauses- agg-refs (->> (:aggregation stage)- (filter vector?)- (keep (fn [clause]- (let [tag (first clause)]- (case (str tag)- "measure" {:kind :measures :ref (last clause)}- "metric" {:kind :metrics :ref (last clause)}- nil)))))- ;; Segments from filter clauses- seg-refs (->> (:filters stage)- (filter vector?)- (keep (fn [clause]- (when (= "segment" (str (first clause)))- {:kind :segments :ref (last clause)}))))]- (cond-> acc- (seq snippets) (update :snippets into snippets)- (seq agg-refs) (as-> a (reduce (fn [m {:keys [kind ref]}]- (update m kind (fnil conj []) ref))- a agg-refs))- (seq seg-refs) (update :segments into (map :ref seg-refs)))))- {}- (or stages []))))--;;; ===========================================================================-;;; Type-specific checks-;;; ===========================================================================--(defn- check-card-specific- "Card-specific checks: query validation via deps.analysis + ref extraction."- [store provider data]- (let [card-id (:id data)- card-meta (provider/card-metadata store data)- resolution-failures (:provider/resolution-failures card-meta)- errors (when-not (:provider/missing-database card-meta)- (try- (deps.analysis/check-entity provider :card card-id)- (catch Exception e- #{{:type :validation-exception-error :message (.getMessage e)}})))- {:keys [bad-refs native-errors]} (partition-errors errors)- native-errors (humanize-errors store native-errors)- bad-refs (mapv #(humanize-source-entity store %) bad-refs)- query-refs (try- (let [card (lib.metadata/card provider card-id)- query (lib/query provider (:dataset-query card))]- (extract-refs-from-query store query provider #{card-id}))- (catch Exception _ nil))- entity-refs (extract-entity-refs-from-raw-data data)- refs (when query-refs- (cond-> query-refs- (seq (:snippets entity-refs)) (assoc :snippets (vec (distinct (:snippets entity-refs))))- (seq (:measures entity-refs)) (assoc :measures (vec (:measures entity-refs)))- (seq (:metrics entity-refs)) (assoc :metrics (vec (:metrics entity-refs)))- (seq (:segments entity-refs)) (assoc :segments (vec (:segments entity-refs)))))]- {:card-id card-id- :unresolved (into (or resolution-failures [])- (when (:provider/missing-database card-meta)- [{:type :database :name (:provider/missing-database card-meta)}]))- :refs refs- :bad-refs bad-refs- :native-errors native-errors- :error (when (:provider/missing-database card-meta)- (str "Unknown database: " (:provider/missing-database card-meta)))}))--;;; ===========================================================================-;;; Unified entity checking-;;; ===========================================================================--(defn- check-entity- "Check a single entity by kind. Runs common checks on all entities,- plus type-specific checks (query validation for cards/transforms,- structural checks for dashboards/documents).- Returns [entity-id result-map]."- [store provider kind entity-id]- (try- (let [load-fn (or (kind->load-fn kind) store/load-card!)- data (load-fn store entity-id)- _ (provider/set-database! provider (provider/database-name-for-entity store kind data))- common (check-common store data)- type-specific (case kind- :card (check-card-specific store provider data)- :dashboard {:unresolved (check-dashboard store data)}- :document {:unresolved (check-document store data)}- :transform (check-transform store provider entity-id)- {})- all-unresolved (into common (:unresolved type-specific))- result (cond-> {:name (or (:name data) entity-id)- :entity-id entity-id- :unresolved all-unresolved- :native-errors (:native-errors type-specific)- :bad-refs (:bad-refs type-specific)- :error (:error type-specific)- :sql (:sql type-specific)}- (:card-id type-specific) (assoc :card-id (:card-id type-specific))- (:refs type-specific) (assoc :refs (:refs type-specific))- (not= kind :card) (assoc :kind kind))]- [entity-id result])- (catch Exception e- [entity-id {:name entity-id- :entity-id entity-id- :kind (when (not= kind :card) kind)- :error (.getMessage e)}])- (finally- (provider/clear-database! provider))))--(defn- check-duplicate-entity-ids- "Check for duplicate entity_ids in the index. Returns a map of- synthetic key → result for each duplicate group."- [index]- (when-let [dupes (:duplicates index)]- (into {}- (for [{:keys [kind ref files]} dupes]- [(str "duplicate:" ref)- {:name (str "Duplicate " (name kind) " entity_id: " ref)- :entity-id ref- :kind kind- :error (str "entity_id " ref " appears in " (count files) " files: "- (str/join ", " files))}]))))--(def ^:private checked-kinds- "Entity kinds that the checker validates, in processing order."- [:card :dashboard :collection :document :measure :segment :transform])--(defn check-entities- "Check all entities: cards, dashboards, collections, documents, transforms, etc.- Returns `{entity-id result-map}`. See the Results processing section for the- result-map shape.-- `schema-source` — a SchemaSource for resolving databases/tables/fields- `assets-source` — an AssetsSource for resolving cards/snippets/transforms/segments- `index` — a file index (see store/make-store)- `entity-ids` — optional seq of entity-ids to check (defaults to all entities)"- ([schema-source assets-source index]- (check-entities schema-source assets-source index nil))- ([schema-source assets-source index entity-ids]- (binding [mu.fn/*enforce* false- qp.i/*skip-middleware-because-app-db-access* true]- (let [store (store/make-store schema-source assets-source index)- provider (provider/make-provider store)- results (if entity-ids- ;; Check specific entities — look up their kind- (into {}- (for [eid entity-ids- :let [kind (or (store/index-kind-of store eid) :card)]]- (check-entity store provider kind eid)))- ;; Check all entities by kind- (into {}- (for [kind checked-kinds- eid (store/all-refs store kind)]- (check-entity store provider kind eid))))- dupes (check-duplicate-entity-ids index)]- (merge results dupes)))))--;;; ===========================================================================-;;; Results processing — pure functions on result data-;;;-;;; Result map shape (per entity):-;;;-;;; :name string — entity name-;;; :entity-id string — portable entity ID-;;; :card-id int? — synthetic integer ID (cards only)-;;; :kind keyword? — :dashboard, :transform, etc. (non-card entities)-;;; :error string? — fatal error message (e.g. unknown database)-;;; :unresolved [map] — unresolved reference failures-;;; each {:type kw, :path vec?, :entity-id str?,-;;; :name str?, :message str?}-;;; :bad-refs [map] — query issues from deps.analysis-;;; (e.g. {:type :validation-exception-error, :message str})-;;; :native-errors #{map} — native SQL errors from deps.analysis-;;; (e.g. {:type :missing-column, :name str,-;;; :source-entity-type kw?, :source-entity-id str?})-;;; :refs map? — extracted references for display (cards only)-;;; {:tables [str], :fields [str], :source-cards [str]}-;;; :sql string? — source SQL (transforms with native queries)-;;;-;;; Summary map shape (from summarize-results):-;;;-;;; :total int — total entities checked-;;; :ok int — entities with no issues-;;; :errors int — entities with fatal errors-;;; :unresolved int — entities with unresolved references-;;; :native-errors int — entities with native SQL errors-;;; :issues int — entities with bad refs / query issues-;;; ===========================================================================--(defn result-status- "Compute the status of a single entity result.- Returns :error, :unresolved, :native-errors, :issues, or :ok (checked in priority order)."- [result]- (cond- (:error result) :error- (seq (:unresolved result)) :unresolved- (seq (:native-errors result)) :native-errors- (seq (:bad-refs result)) :issues- :else :ok))--(defn summarize-results- "Summarize check results into counts by status."- [results]- (let [by-status (group-by (comp result-status second) results)]- {:total (count results)- :ok (count (get by-status :ok))- :errors (count (get by-status :error))- :unresolved (count (get by-status :unresolved))- :native-errors (count (get by-status :native-errors))- :issues (count (get by-status :issues))}))--(defn results-by-status- "Group results by status. Returns map of status → seq of [entity-id result]."- [results]- (group-by (comp result-status second) results))--(defn format-result- "Format a single entity result as a human-readable string."- [[entity-id result]]- (let [lines (atom [(str "=== " (:name result) " [" entity-id "] ===")- (if-let [kind (:kind result)]- (str " Kind: " (name kind))- (str " Card ID: " (:card-id result)))])- {:keys [tables fields source-cards]} (:refs result)]- (when (seq tables)- (swap! lines conj (str " Tables: " (str/join ", " tables))))- (when (seq fields)- (swap! lines conj (str " Fields: " (str/join ", " fields))))- (when (seq source-cards)- (swap! lines conj (str " Source Cards: " (str/join ", " source-cards))))- (when-let [unresolved (:unresolved result)]- (swap! lines conj " UNRESOLVED REFERENCES:")- (doseq [{:keys [type path entity-id name message]} unresolved]- (swap! lines conj (str " - " (clojure.core/name type) ": "- (or (some->> path (str/join ".")) entity-id name message)))))- (when (seq (:native-errors result))- (swap! lines conj " NATIVE SQL ERRORS:")- (doseq [err (:native-errors result)]- (swap! lines conj (str " - " (pr-str err))))- (when-let [sql (:sql result)]- (swap! lines conj (str " SQL: " sql))))- (swap! lines conj (case (result-status result)- :error (str " ERROR: " (:error result))- :unresolved " Status: MISSING REFS"- :native-errors " Status: NATIVE SQL ERRORS"- :issues (str " Status: ISSUES FOUND\n"- (str/join "\n" (map #(str " - " (pr-str %)) (:bad-refs result))))- :ok " Status: OK"))- (str/join "\n" @lines)))--(defn- issue-line- "Render a single issue as ` - <type>: <detail>`. Skips refs that are nil- and uses the first non-empty detail field."- [{:keys [type path entity-id name message]}]- (let [type-name (clojure.core/name type)- type-label (-> type-name- (str/replace #"-" " "))- detail (or (some->> path (str/join "."))- entity-id- name- message)]- (str " - "- (case type- :duplicate-column (str "duplicate column: " name)- :missing-column (str "missing column: " name)- :missing-table-alias (str "missing table alias: " name)- :syntax-error "syntax error"- :validation-exception-error (str "validation error: " message)- ;; unresolved-ref types — prefix with "unresolved"- (:database :table :field :card :snippet :transform :segment- :measure :dashboard :collection :document)- (str "unresolved " type-name ": " detail)- ;; structural checks (e.g. :dashcard-grid, :container-conflict)- (str type-label ": " (or detail ""))))))--(defn format-fail- "Format a single failed entity result as a human-readable block.- Returns nil for :ok results.-- FAIL <kind> \"<name>\" [<entity-id>]- - issue 1- - issue 2- ..."- [[entity-id result]]- (let [status (result-status result)]- (when (not= :ok status)- (let [kind-label (if-let [kind (:kind result)] (clojure.core/name kind) "card")- issues (concat (:unresolved result)- (:bad-refs result)- (:native-errors result)- (when-let [err (:error result)]- [{:type :error :message err}]))- header (str "FAIL " kind-label " \"" (:name result) "\" [" entity-id "]")- body (map issue-line issues)]- (str/join "\n" (cons header body))))))--(defn format-error- "Format a single card error concisely for LLM consumption.- Returns nil for :ok results. Only includes actionable error information."- [[entity-id result]]- (let [status (result-status result)]- (when (not= :ok status)- (let [kind-label (if-let [kind (:kind result)] (name kind) "card")- lines (atom [(str kind-label ": " (:name result) " (entity_id: " entity-id ")")])]- (when-let [unresolved (seq (:unresolved result))]- (doseq [{:keys [type path entity-id name message]} unresolved]- (swap! lines conj (str " unresolved " (clojure.core/name type) ": "- (or (some->> path (str/join ".")) entity-id name message)))))- (when (seq (:bad-refs result))- (doseq [ref (:bad-refs result)]- (swap! lines conj (str " bad ref: " (pr-str ref)))))- (when (seq (:native-errors result))- (doseq [err (:native-errors result)]- (swap! lines conj (str " sql error: " (pr-str (dissoc err :source-entity-type :source-entity-id)))))- (when-let [sql (:sql result)]- (swap! lines conj (str " sql: " sql))))- (when (:error result)- (swap! lines conj (str " error: " (:error result))))- (str/join "\n" @lines)))))--(defn- make-schema-source- "Construct a schema source. `schema-format` is one of :serdes or :concise.- For :serdes, `schema-path` is a directory containing database subdirectories.- For :concise, `schema-path` is a single JSON file with the concise metadata."- [schema-format schema-path]- (case schema-format- :concise (concise-schema/make-source schema-path)- :serdes (serdes.schema/make-database-source schema-path)- (throw (ex-info (str "Unknown schema format: " (pr-str schema-format)- ". Must be :serdes or :concise.")- {:schema-format schema-format}))))--(defn- make-sources-and-index- "Build schema and assets sources, plus the merged assets index (cards,- dashboards, segments, etc.). `opts` may contain `:schema-format` (defaults- to :serdes)."- [export-dir schema-path & {:keys [schema-format] :or {schema-format :serdes}}]- (let [schema-source (make-schema-source schema-format schema-path)- assets-source (serdes-assets/make-source export-dir)- index (serdes-assets/source-index assets-source)]- {:schema-source schema-source :assets-source assets-source :index index}))--(defn check- "Check entities in an export directory against database schemas.-- `export-dir` — directory containing serdes-exported entities (collections/)- `schema-path` — path to schema source (directory for :serdes, file for :concise)- `entity-ids` — optional seq of entity-ids to check (defaults to all entities)- `opts` — optional map; `:schema-format` is :serdes (default) or :concise-- Returns `{:results results-map}` where results-map is `{entity-id result-map}`.- See the Results processing section above for the result-map and summary-map shapes."- ([export-dir schema-path]- (check export-dir schema-path nil nil))- ([export-dir schema-path entity-ids]- (check export-dir schema-path entity-ids nil))- ([export-dir schema-path entity-ids opts]- (let [{:keys [schema-source assets-source index]}- (make-sources-and-index export-dir schema-path- :schema-format (:schema-format opts :serdes))]- {:results (if entity-ids- (check-entities schema-source assets-source index entity-ids)- (check-entities schema-source assets-source index))})))--(defn setup- "Create a store and provider for REPL iteration.- Returns {:store store :provider provider :index index}.-- `opts` may contain `:schema-format` (:serdes default, or :concise).-- Usage:- (def ctx (setup \"/path/to/export\" \"/path/to/schemas\"))- (def ctx (setup \"/path/to/export\" \"/path/to/metadata.json\" {:schema-format :concise}))- (check-one ctx \"entity-id\")- (check-one ctx \"entity-id\" :verbose true)"- ([export-dir schema-path]- (setup export-dir schema-path nil))- ([export-dir schema-path opts]- (let [{:keys [schema-source assets-source index]}- (make-sources-and-index export-dir schema-path- :schema-format (:schema-format opts :serdes))- store (binding [mu.fn/*enforce* false]- (store/make-store schema-source assets-source index))- provider (provider/make-provider store)]- {:store store :provider provider :index index})))--(defn check-one- "Check a single entity by entity-id using a pre-built context from [[setup]].- Looks up the entity's kind from the store. Returns the result map.- With :verbose true, also prints formatted output."- [{:keys [store provider]} entity-id & {:keys [verbose]}]- (binding [mu.fn/*enforce* false- qp.i/*skip-middleware-because-app-db-access* true]- (let [kind (or (store/index-kind-of store entity-id) :card)- [_ result] (check-entity store provider kind entity-id)]- (when verbose- #_{:clj-kondo/ignore [:discouraged-var]}- (println (format-result [entity-id result])))- result)))--(comment- ;; REPL workflow:- (check "/Users/dan/projects/work/stats-remote-sync"- "/Users/dan/projects/work/exports-root/metadata/metadata/databases")-- (->> (check "/Users/dan/projects/work/representations/examples/v1"- "/Users/dan/projects/work/yaml-checked-files-v1/exports/sqlite-based/databases")- (into {} (filter (fn [[_id stuff]]- (letfn [(bad [x] ((some-fn :bad-refs :unresolved :native-errors) x))]- (bad stuff))))))- (setup- "/Users/dan/projects/work/stats-remote-sync"- "/Users/dan/projects/work/stats-remote-sync/databases")-- (check-one (setup "/Users/dan/projects/work/stats-remote-sync"- "/tmp/metadata/metadata/databases")- "2MNFkdc6EvgmUJU4E0ytF"))diff --git a/enterprise/backend/src/metabase_enterprise/checker/source.clj b/enterprise/backend/src/metabase_enterprise/checker/source.cljdeleted file mode 100644index ddcb2d397b6..00000000000--- a/enterprise/backend/src/metabase_enterprise/checker/source.clj+++ /dev/null@@ -1,204 +0,0 @@-(ns metabase-enterprise.checker.source- "Protocols for resolving portable references to entity data.-- Two protocols separate the two concerns:-- - **SchemaSource** resolves database schema references (databases, tables, fields).- Backed by the `--schema-dir` directory.- - **AssetsSource** resolves serialized entity references (cards, snippets, transforms,- segments, dashboards, collections, documents, measures).- Backed by the `--export` directory.-- The checker assigns integer IDs; sources just say 'yes this exists, here's the- data' or 'no, this reference is unresolved'.-- ## The contract-- A source returns plain Clojure maps with **snake_case keys** matching the- serialized form. The provider (`checker.provider`) is responsible for all- conversion to lib metadata format:-- - snake_case → kebab-case (`:base_type` → `:base-type`)- - strings → keywords (`:engine \"h2\"` → `:engine :h2`)- - adding `:lib/type` discriminators- - filling in defaults (`:active` defaults to `true` if absent)- - resolving portable refs (path vectors) to integer IDs via the store- - MBQL normalization-- This means a source author only has to answer 'given this portable reference,- give me the raw map' — no need to know about lib metadata, type keywords, or- the integer ID space. New sources (REST API, git repo, LLM, app DB via- toucan2) can return the snake_case shape directly.-- ## Portable references-- - Database: string `db-name`- - Table: `[db-name schema-name table-name]` — `schema-name` may be `nil`- for schema-less databases (e.g. SQLite)- - Field: `[db-name schema-name table-name field-name]`- - Card / snippet / transform / segment / dashboard / collection / document /- measure: string `entity-id`-- ## Required vs optional keys-- The docstrings below describe each `resolve-*` method's return shape. Keys- marked **required** must be present for the checker to function. Keys marked- **optional** are read defensively (the provider treats them as nil/default- when absent).")--(set! *warn-on-reflection* true)--(defprotocol SchemaSource- "Resolve database schema references to entity data."-- (resolve-database [this db-name]- "Resolve a database by its real name. Returns a map or nil.-- Returned map keys (snake_case):- :id any — required; opaque to checker, used as synthetic ID- :name string — required- :engine string — required; provider keywordizes (e.g. \"h2\" → :h2)- :settings map — optional- :dbms_version map — optional")-- (resolve-table [this table-path]- "Resolve a table by `[db-name schema-name table-name]`. `schema-name` is nil- for schema-less databases. Returns a map or nil.-- Returned map keys (snake_case):- :id any — required; opaque to checker- :name string — required- :display_name string — optional- :schema string — required (may be nil for schema-less DBs)- :db_id any — required; matches the database :id (or db-name)- :active bool — optional, defaults to true- :visibility_type string — optional; provider keywordizes")-- (resolve-field [this field-path]- "Resolve a field by `[db-name schema-name table-name field-name]`.- Returns a map or nil.-- Returned map keys (snake_case):- :id any — required; opaque to checker- :table_id any — required; matches the table :id (or table-path)- :name string — required- :display_name string — optional- :base_type string — required; provider keywordizes- :effective_type string — optional; defaults to :base_type- :semantic_type string — optional; provider keywordizes- :database_type string — optional- :active bool — optional, defaults to true- :visibility_type string — optional; provider keywordizes- :position int — optional- :fk_target_field_id vector — optional; must be a field-path vector- to participate in FK resolution")-- (fields-for-table [this table-path]- "Return a collection of field-paths belonging to the given table-path- `[db-name schema-name table-name]`. Returns nil if the table is unknown.")-- (all-field-paths [this]- "Return a collection of all known field-paths.")-- (all-database-names [this]- "Return a collection of all known database names (strings).")-- (all-table-paths [this]- "Return a collection of all known table-paths.")-- (tables-for-database [this db-name]- "Return a collection of table-paths belonging to the given database name."))--(defprotocol AssetsSource- "Resolve serialized entity references to entity data.-- Each `resolve-*` method takes an `entity-id` string and returns the raw- entity map (or nil if not found). Maps use snake_case keys matching the- serialized YAML/JSON form."-- (resolve-card [this entity-id]- "Resolve a card by entity-id. Returns a map or nil.-- Returned map keys (snake_case):- :id any — required; opaque to checker- :name string — required- :type string — required; provider keywordizes (e.g. \"question\")- :dataset_query map — required; MBQL query, normalized by provider- Has :database (db-name string or db-id),- :stages (or :type+:query for legacy form)- :table_id vector|int — optional; portable path or integer ID- :result_metadata seq — optional; column maps, may contain portable refs- in :id, :table_id, :fk_target_field_id, :field_ref- :archived bool — optional- :collection_id string — optional; entity-id of containing collection- :dashboard_id string — optional; entity-id of containing dashboard- :document_id string — optional; entity-id of containing document")-- (resolve-snippet [this entity-id]- "Resolve a native query snippet by entity-id. Returns a map or nil.-- Returned map keys (snake_case):- :id any — required; opaque to checker- :name string — required- :content string — required; the snippet's SQL fragment")-- (resolve-transform [this entity-id]- "Resolve a transform by entity-id. Returns a map or nil.-- Returned map keys (snake_case):- :id any — required; opaque to checker- :name string — required- :source map — required- Has :type (\"query\" for query-based transforms),- and :query when type=\"query\"- :source_database_id any — optional; fallback when :source.query has no :database")-- (resolve-segment [this entity-id]- "Resolve a segment by entity-id. Returns a map or nil.-- Returned map keys (snake_case):- :id any — required; opaque to checker- :name string — required- :definition map — required; MBQL definition (a dataset_query)")-- (resolve-dashboard [this entity-id]- "Resolve a dashboard by entity-id. Returns a map or nil.-- Returned map keys (snake_case):- :id any — required; opaque to checker- :name string — required- :tabs seq — required; each tab has :entity_id (string)- :dashcards seq — required; each dashcard has:- :entity_id string — required- :card_id string — optional (nil for virtual cards)- :dashboard_tab_id vector|string — optional- :col, :row,- :size_x, :size_y int — optional, default 0/0/1/1- :collection_id string — optional; entity-id of containing collection")-- (resolve-collection [this entity-id]- "Resolve a collection by entity-id. Returns a map or nil.-- Returned map keys (snake_case):- :id any — required; opaque to checker- :name string — required- :parent_id string — optional; entity-id of parent collection- :collection_id string — optional; same as :parent_id in some contexts")-- (resolve-document [this entity-id]- "Resolve a document by entity-id. Returns a map or nil.-- Returned map keys (snake_case):- :id any — required; opaque to checker- :name string — required- :document map — required; ProseMirror content tree- cardEmbed nodes have :attrs.id with serdes refs- text nodes may have :marks with link hrefs- :collection_id string — optional; entity-id of containing collection")-- (resolve-measure [this entity-id]- "Resolve a measure by entity-id. Returns a map or nil.-- Returned map keys (snake_case):- :id any — required; opaque to checker- :name string — required"))diff --git a/enterprise/backend/src/metabase_enterprise/checker/store.clj b/enterprise/backend/src/metabase_enterprise/checker/store.cljdeleted file mode 100644index 9d7b4517eb4..00000000000--- a/enterprise/backend/src/metabase_enterprise/checker/store.clj+++ /dev/null@@ -1,267 +0,0 @@-(ns metabase-enterprise.checker.store- "The store knows what entities exist and can load them.-- It combines three things:- - A **SchemaSource** for resolving databases, tables, and fields- - An **AssetsSource** for resolving cards, snippets, transforms, and segments- - A **file index** that enumerates all known entities by kind and ref-- On top of that it maintains:- - A bidirectional **ID registry** (portable refs ↔ synthetic integer IDs,- because lib requires integer IDs)- - **Entity caches** (loaded data with :id stamped on)-- Entities are loaded lazily from sources and cached on first access.- The store is an atom passed explicitly — no dynamic vars."- (:require- [metabase-enterprise.checker.source :as source]))--(set! *warn-on-reflection* true)--;;; ===========================================================================-;;; Store creation-;;; ===========================================================================--(defn make-store- "Create a fresh store for a checking session.-- `schema-source` is a SchemaSource for resolving databases/tables/fields.- `assets-source` is an AssetsSource for resolving cards/snippets/transforms/segments.- `index` is a file index: `{kind {ref file-path}}` — databases from the schema- source, cards/dashboards/collections/transforms/segments/measures from assets."- [schema-source assets-source index]- (atom {:schema-source schema-source- :assets-source assets-source- :index index- :id-counter 0- :ref->id {} ; {kind {ref id}}- :id->ref {} ; {kind {id ref}}- :entities {}})) ; {kind {ref data}}--;;; ===========================================================================-;;; Index queries-;;; ===========================================================================--(defn all-refs- "All known refs of a given kind from the file index."- [store kind]- (keys (get-in @store [:index kind])))--(defn exists?- "Is `ref` of `kind` known to the store?- Databases are checked via the schema source (not the assets index)."- [store kind ref]- (if (= kind :database)- (contains? (set (source/all-database-names (:schema-source @store))) ref)- (contains? (get-in @store [:index kind]) ref)))--(defn index-kind-of- "Return the kind of an entity-id in the index, or nil if not found."- [store entity-id]- (cond- (exists? store :collection entity-id) :collection- (exists? store :card entity-id) :card- (exists? store :dashboard entity-id) :dashboard- (exists? store :document entity-id) :document- (exists? store :measure entity-id) :measure- (exists? store :segment entity-id) :segment- (exists? store :snippet entity-id) :snippet- (exists? store :transform entity-id) :transform- :else nil))--(defn index-file- "Get the file path for a ref of a given kind from the index, or nil."- [store kind ref]- (get-in @store [:index kind ref]))--(defn all-database-names- "All database names from the schema source."- [store]- (source/all-database-names (:schema-source @store)))--(defn all-table-paths- "All table paths from the schema source."- [store]- (source/all-table-paths (:schema-source @store)))--(defn all-field-paths- "All field paths from the schema source."- [store]- (source/all-field-paths (:schema-source @store)))--(defn all-card-ids- "All card entity-ids from the file index."- [store]- (all-refs store :card))--(defn tables-for-database- "Table paths belonging to a specific database."- [store db-name]- (source/tables-for-database (:schema-source @store) db-name))--(defn fields-for-table- "Field paths belonging to a specific table path."- [store table-path]- (source/fields-for-table (:schema-source @store) table-path))--;;; ===========================================================================-;;; ID registry — assign synthetic integer IDs to portable refs-;;; ===========================================================================--(defn- next-id! [store]- (:id-counter (swap! store update :id-counter inc)))--(defn get-or-assign!- "Return the integer ID for `ref` under `kind`, assigning one if needed."- [store kind ref]- (or (get-in @store [:ref->id kind ref])- (let [id (next-id! store)]- (swap! store (fn [s]- (-> s- (assoc-in [:ref->id kind ref] id)- (assoc-in [:id->ref kind id] ref))))- id)))--(defn id->ref- "Look up the portable ref for an integer ID. Returns nil if unknown."- [store kind id]- (get-in @store [:id->ref kind id]))--(defn ref->id- "Look up the integer ID for a portable ref. Returns nil if not yet assigned."- [store kind ref]- (get-in @store [:ref->id kind ref]))--;;; ===========================================================================-;;; Entity loading — lazy load from source, cache with assigned IDs-;;; ===========================================================================--(defn- schema-source [store]- (:schema-source @store))--(defn- assets-source [store]- (:assets-source @store))--(defn cached-entity- "Look up a cached entity by kind and ref. Returns nil if not cached."- [store kind ref]- (get-in @store [:entities kind ref]))--(defn- cache-entity! [store kind ref data]- (swap! store assoc-in [:entities kind ref] data)- data)--;; --- Schema entities -----(defn load-database!- "Load and cache a database, assigning it an integer ID. Returns data or nil."- [store db-name]- (or (cached-entity store :database db-name)- (when-let [data (source/resolve-database (schema-source store) db-name)]- (let [id (get-or-assign! store :database db-name)]- (cache-entity! store :database db-name (assoc data :id id))))))--(defn ensure-table-id!- "Ensure a table path has an assigned integer ID and minimal metadata cached.- Does NOT parse the table YAML — derives name/schema from the path.- Use [[load-table!]] when full table data is needed."- [store [db-name schema table-name :as table-path]]- (or (cached-entity store :table table-path)- (let [id (get-or-assign! store :table table-path)- db-id (get-or-assign! store :database db-name)]- (cache-entity! store :table table-path- {:id id- :db_id db-id- :name table-name- :schema schema}))))--(defn load-table!- "Load and cache a table, assigning it and its database integer IDs.- Parses the table YAML for full metadata."- [store table-path]- (let [cached (cached-entity store :table table-path)]- (if (and cached (contains? cached :display_name))- ;; Already fully loaded- cached- ;; Load from YAML, merging with any existing minimal cache- (when-let [data (source/resolve-table (schema-source store) table-path)]- (let [[db-name _ _] table-path- id (get-or-assign! store :table table-path)- db-id (get-or-assign! store :database db-name)]- (cache-entity! store :table table-path (assoc data :id id :db_id db-id)))))))--(defn load-field!- "Load and cache a field, assigning it and its table integer IDs."- [store field-path]- (or (cached-entity store :field field-path)- (when-let [data (source/resolve-field (schema-source store) field-path)]- (let [[db-name schema table-name _] field-path- id (get-or-assign! store :field field-path)- table-id (get-or-assign! store :table [db-name schema table-name])]- (cache-entity! store :field field-path (assoc data :id id :table_id table-id))))))--;; --- Asset entities -----(defn load-card!- "Load and cache a card, assigning it an integer ID."- [store entity-id]- (or (cached-entity store :card entity-id)- (when-let [data (source/resolve-card (assets-source store) entity-id)]- (let [id (get-or-assign! store :card entity-id)]- (cache-entity! store :card entity-id (assoc data :id id))))))--(defn load-snippet!- "Load and cache a snippet, assigning it an integer ID."- [store entity-id]- (or (cached-entity store :snippet entity-id)- (when-let [data (source/resolve-snippet (assets-source store) entity-id)]- (let [id (get-or-assign! store :snippet entity-id)]- (cache-entity! store :snippet entity-id (assoc data :id id))))))--(defn load-transform!- "Load and cache a transform, assigning it an integer ID."- [store entity-id]- (or (cached-entity store :transform entity-id)- (when-let [data (source/resolve-transform (assets-source store) entity-id)]- (let [id (get-or-assign! store :transform entity-id)]- (cache-entity! store :transform entity-id (assoc data :id id))))))--(defn load-segment!- "Load and cache a segment, assigning it an integer ID."- [store entity-id]- (or (cached-entity store :segment entity-id)- (when-let [data (source/resolve-segment (assets-source store) entity-id)]- (let [id (get-or-assign! store :segment entity-id)]- (cache-entity! store :segment entity-id (assoc data :id id))))))--(defn load-dashboard!- "Load and cache a dashboard, assigning it an integer ID."- [store entity-id]- (or (cached-entity store :dashboard entity-id)- (when-let [data (source/resolve-dashboard (assets-source store) entity-id)]- (let [id (get-or-assign! store :dashboard entity-id)]- (cache-entity! store :dashboard entity-id (assoc data :id id))))))--(defn load-collection!- "Load and cache a collection, assigning it an integer ID."- [store entity-id]- (or (cached-entity store :collection entity-id)- (when-let [data (source/resolve-collection (assets-source store) entity-id)]- (let [id (get-or-assign! store :collection entity-id)]- (cache-entity! store :collection entity-id (assoc data :id id))))))--(defn load-document!- "Load and cache a document, assigning it an integer ID."- [store entity-id]- (or (cached-entity store :document entity-id)- (when-let [data (source/resolve-document (assets-source store) entity-id)]- (let [id (get-or-assign! store :document entity-id)]- (cache-entity! store :document entity-id (assoc data :id id))))))--(defn load-measure!- "Load and cache a measure, assigning it an integer ID."- [store entity-id]- (or (cached-entity store :measure entity-id)- (when-let [data (source/resolve-measure (assets-source store) entity-id)]- (let [id (get-or-assign! store :measure entity-id)]- (cache-entity! store :measure entity-id (assoc data :id id))))))diff --git a/enterprise/backend/src/metabase_enterprise/core/init.clj b/enterprise/backend/src/metabase_enterprise/core/init.cljindex 22575758139..1abe35a39d1 100644--- a/enterprise/backend/src/metabase_enterprise/core/init.clj+++ b/enterprise/backend/src/metabase_enterprise/core/init.clj@@ -25,7 +25,6 @@[metabase-enterprise.scim.init][metabase-enterprise.security-center.init][metabase-enterprise.semantic-search.init]- [metabase-enterprise.serialization.init][metabase-enterprise.sso.init][metabase-enterprise.stale.init][metabase-enterprise.support-access-grants.init]diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.cljindex b3f4ac6fc73..f99fd8bb223 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/cli.clj@@ -25,7 +25,7 @@java -jar metabase.jar --mode complexity-score \\--source representation -r path/to/fixture/- Patterned after [[metabase-enterprise.checker.cli]] — same `fail!` pattern, same arg style,+ Same `fail!` pattern and arg style as the other standalone-mode CLIs,and the same bootstrap-dispatched [[entrypoint]] hook."(:require[clojure.java.io :as io]diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/representation.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/representation.cljindex f9de9260a8a..975ed85c8c8 100644--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/representation.clj+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/representation.clj@@ -2,7 +2,7 @@"Load a serdes export directory into the shape the complexity scorer expects.Format: the official Metabase serdes/representation YAML, byte-compatible with what- `serdes/store` emits and what `metabase-enterprise.checker` parses.+ `serdes/store` emits.Layout (subset that the scorer reads):diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/core.cljindex b98aadbe30c..aa84c79a93e 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj@@ -1,7 +1,6 @@(ns metabase-enterprise.remote-sync.core(:require[metabase-enterprise.remote-sync.guards :as guards]- [metabase-enterprise.remote-sync.impl :as impl][metabase-enterprise.remote-sync.settings :as settings][metabase-enterprise.remote-sync.source :as source][metabase-enterprise.remote-sync.source.protocol :as source.p]@@ -20,49 +19,7 @@(p/import-vars[source][source.p- ->ingestable]- [settings- remote-sync-enabled- remote-sync-url- remote-sync-branch- remote-sync-token])--(defn start-import!- "Start an asynchronous import from the configured remote-sync source,- regardless of `remote-sync-type`. Returns the started RemoteSyncTask, or nil- when there is nothing new to import. Throws when remote sync is not- configured, another sync task is already running, or the source is- unreachable."- []- (let [branch (settings/remote-sync-branch)]- (impl/async-import! branch true {}- :on-success (fn [task-id _result]- (impl/publish-sync-event! :event/remote-sync-import task-id- {:branch branch :auto true} nil)))))--(defn branch-exists?- "Whether `branch-name` exists on the configured remote. Throws when remote- sync is not configured or the remote is unreachable."- [branch-name]- (contains? (set (source.p/branches (source/source-from-settings))) branch-name))--(defn create-branch!- "Create `branch-name` on the configured remote, branching off the configured- remote-sync branch (or the remote's default branch when none is- configured). Throws when the branch already exists, remote sync is not- configured, or the remote is unreachable."- [branch-name]- (let [source (source/source-from-settings)- base (or (not-empty (settings/remote-sync-branch))- (source.p/default-branch source))]- (source.p/create-branch source branch-name base)))--(defn delete-branch!- "Delete `branch-name` on the configured remote. A no-op when the branch does- not exist. Throws when remote sync is not configured or the remote refuses- the deletion."- [branch-name]- (source.p/delete-branch (source/source-from-settings) branch-name))+ ->ingestable])(defenterprise collection-editable?"Determines if a remote-synced collection should be editable.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.cljindex eeb89689010..05b151d0351 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj@@ -494,35 +494,6 @@(push-branch! (assoc source :branch branch-name))branch-name))-(defn delete-branch- "Deletes a branch from the remote repository, along with the local ref.- A no-op when the branch does not exist on the remote.-- Takes a source map containing a :git Git instance and optional :token for- authentication, and the branch-name string to delete.-- Returns nil. Throws ExceptionInfo if the remote refuses the deletion."- [{:keys [^Git git] :as source} branch-name]- (let [branch-ref (qualify-branch branch-name)- push-cmd (-> (.push git)- (.setRefSpecs (doto (java.util.ArrayList.)- ;; empty source side = delete the remote ref- (.add (RefSpec. (str ":" branch-ref))))))- push-response (call-remote-command push-cmd source)- statuses (->> push-response- (mapcat #(.getRemoteUpdates ^PushResult %))- (map #(.getStatus ^RemoteRefUpdate %)))]- (when-let [failures (seq (remove #{RemoteRefUpdate$Status/OK- RemoteRefUpdate$Status/UP_TO_DATE- RemoteRefUpdate$Status/NON_EXISTING}- statuses))]- (throw (ex-info (str "Failed to delete branch " branch-name " on remote") {:failures failures})))- (when (.resolve (.getRepository git) branch-ref)- (call-command (-> (.branchDelete git)- (.setBranchNames ^"[Ljava.lang.String;" (into-array String [branch-name]))- (.setForce true))))- nil))-(defrecord GitSnapshot [git remote-url branch version token managed-dirs]source.p/SourceSnapshot@@ -607,9 +578,6 @@(create-branch [source branch-name base-commit-ish](create-branch source branch-name base-commit-ish))- (delete-branch [source branch-name]- (delete-branch source branch-name))-(default-branch [this](default-branch this))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.cljindex 74c0aa2c072..342e3de2b43 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj@@ -18,13 +18,6 @@Returns the name of the newly created branch.")- (delete-branch [source branch-name]- "Deletes a branch from the remote source. A no-op when the branch does not exist.-- Takes a source instance implementing this protocol and the branch-name to delete.-- Returns nil.")-(default-branch [source]"Gets the default branch name from the remote source.diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.cljindex 1c0184dc95d..5ba8af2f0d3 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj@@ -3,9 +3,6 @@[clojure.java.io :as io][clojure.string :as str][java-time.api :as t]- [metabase-enterprise.serialization.export :as export]- [metabase-enterprise.serialization.metadata-file-import :as metadata-file-import]- [metabase-enterprise.serialization.schema :as schema][metabase-enterprise.serialization.v2.extract :as extract][metabase-enterprise.serialization.v2.ingest :as v2.ingest][metabase-enterprise.serialization.v2.load :as v2.load]@@ -27,10 +24,9 @@[metabase.util.malli :as mu][metabase.util.malli.schema :as ms][metabase.util.random :as u.random]- [ring.core.protocols :as ring.protocols]- [ring.util.response :as response])+ [ring.core.protocols :as ring.protocols])(:import- (java.io ByteArrayOutputStream File InputStream)))+ (java.io ByteArrayOutputStream File)))(set! *warn-on-reflection* true)@@ -340,107 +336,6 @@(finally(io/delete-file (:tempfile file)))))-;;; ----------------------------------- POST /api/ee/serialization/metadata/export -------------------------------------(api.macros/defendpoint :post "/metadata/export"- :- (sr/streaming-response-schema ::schema/export-metadata-response)- "Get warehouse metadata (databases, tables, and fields) for all databases visible to the- current user. References between rows are emitted as raw numeric ids (`db_id`,- `table_id`, `parent_id`, `fk_target_field_id`).-- Sections must be opted into with the `with-databases`, `with-tables`, and `with-fields`- query parameters — they all default to `false`. The response is streamed for efficiency- with large schemas.-- Requires `View data` → `Can view` and `Create queries` → `Query builder only` (or- `Query builder and native`) permissions on each database and table."- [_route-params- query-params :- [:map- [:with-databases {:default false} [:maybe :boolean]]- [:with-tables {:default false} [:maybe :boolean]]- [:with-fields {:default false} [:maybe :boolean]]]]- (let [opts (assoc query-params :user-info {:user-id api/*current-user-id*- :is-superuser? api/*is-superuser?*})]- (sr/streaming-response {:content-type "application/json; charset=utf-8"} [os _]- (export/export-metadata! os opts))))--;;; ----------------------------------- POST /api/ee/serialization/metadata/import -------------------------------------(defn- spool-to-temp-file!- "Stream `is` to a freshly-created temp file in 64 KB chunks. Returns the- `File`. If streaming fails the temp file is removed before the throw- propagates."- ^File [^InputStream is]- (let [tmp (File/createTempFile "metadata-import-" ".json")]- (try- (with-open [os (io/output-stream tmp)]- (io/copy is os :buffer-size (* 64 1024)))- tmp- (catch Throwable t- (.delete tmp)- (throw t)))))--(api.macros/defendpoint :post "/metadata/import"- :- [:map- [:status [:= 202]]- [:body [:map {:closed true}- [:queued :boolean]- [:import-id :string]]]]- "Import warehouse metadata previously emitted by `POST /metadata/export`. The- request body is the JSON document `{databases, tables, fields}`; sections are- parsed incrementally so memory stays bounded regardless of payload size.-- To bypass the JSON-parsing request middleware, send with `Content-Type:- application/octet-stream`. Restricted to superusers.-- Returns `202` immediately with an `:import-id`; the import runs- asynchronously. Poll `GET /metadata/import/:id` for its outcome."- [_route-params- _query-params- _body- {:keys [body], :as _request}]- (api/check-superuser)- (cond- (nil? body)- (throw (ex-info "Empty request body" {:status-code 400}))-- (not (instance? InputStream body))- (throw (ex-info (str "Expected a raw stream body. Send the request with "- "Content-Type: application/octet-stream so the JSON "- "middleware does not pre-parse the payload.")- {:status-code 415}))-- :else- (let [tmp (spool-to-temp-file! body)- id (metadata-file-import/enqueue-import! tmp {:delete-after? true})]- (-> (response/response {:queued true :import-id id})- (assoc :status 202)))))--;;; ------------------------------- GET /api/ee/serialization/metadata/import/:id ---------------------------------(defn- present-import-status- "Project an internal registry record onto the public wire shape. Deliberately- omits the server-side `:file` path; stringifies the status keyword and the- timestamps."- [record]- {:id (:id record)- :status (name (:status record))- :enqueued-at (str (:enqueued-at record))- :started-at (some-> (:started-at record) str)- :finished-at (some-> (:finished-at record) str)- :wall-ms (:wall-ms record)- :error (:error record)})--(api.macros/defendpoint :get "/metadata/import/:id"- :- ::schema/import-status-response- "Status of a metadata import previously started by `POST /metadata/import`.- Status is retained in-memory and is not durable across server restarts.- Restricted to superusers."- [{:keys [id]} :- [:map [:id ms/UUIDString]]]- (api/check-superuser)- (or (some-> (metadata-file-import/import-status id) present-import-status)- (throw (ex-info "Unknown or expired import id" {:status-code 404}))))-(def ^{:arglists '([request respond raise])} routes"`/api/ee/serialization` routes."(api.macros/ns-handler *ns* +auth))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/core.clj b/enterprise/backend/src/metabase_enterprise/serialization/core.cljindex d4269100354..a1a88a32d9e 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/core.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/core.clj@@ -1,10 +1,6 @@(ns metabase-enterprise.serialization.core(:require[metabase-enterprise.serialization.dump]- [metabase-enterprise.serialization.export]- ;; Required (without alias) so loading `serialization.core` populates the malli- ;; registry — callers can then refer to `::serialization.schema/...` keywords.- [metabase-enterprise.serialization.schema][metabase-enterprise.serialization.v2.extract][metabase-enterprise.serialization.v2.ingest][metabase-enterprise.serialization.v2.load]@@ -15,7 +11,6 @@(commentmetabase-enterprise.serialization.dump/keep-me- metabase-enterprise.serialization.export/keep-memetabase-enterprise.serialization.v2.extract/keep-memetabase-enterprise.serialization.v2.ingest/keep-memetabase-enterprise.serialization.v2.load/keep-me@@ -26,8 +21,6 @@(p/import-vars[metabase-enterprise.serialization.dumpserialization-deep-sort]- [metabase-enterprise.serialization.export- export-metadata!][metabase-enterprise.serialization.v2.extractmake-targets-of-typeextract]diff --git a/enterprise/backend/src/metabase_enterprise/serialization/export.clj b/enterprise/backend/src/metabase_enterprise/serialization/export.cljdeleted file mode 100644index e350714a6da..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/export.clj+++ /dev/null@@ -1,70 +0,0 @@-(ns metabase-enterprise.serialization.export- "Orchestrates the metadata export pipeline: walks [[sections]], runs each- section's `serdes/metadata-query`, formats rows via `serdes/metadata-query-format`,- and streams the result as a JSON object whose keys are the section names.- Sections whose `:enabled?` predicate returns false for `opts` are skipped."- (:require- [metabase-enterprise.serialization.schema :as schema]- [metabase.models.serialization :as serdes]- [metabase.util.json :as json]- [metabase.util.malli :as mu])- (:import- (java.io BufferedWriter OutputStream OutputStreamWriter Writer)- (java.nio.charset StandardCharsets)))--(set! *warn-on-reflection* true)--(def ^:private sections- "Sections emitted by the metadata export, in response order."- [{:name "databases" :model :model/Database :enabled? :with-databases}- {:name "tables" :model :model/Table :enabled? :with-tables}- {:name "fields" :model :model/Field :enabled? :with-fields}])--(defn- write-json-array!- "Streams a reducible collection as a JSON array to `writer`. `run!` is required —- it dispatches through `reduce`, which consumes the `IReduceInit` returned by- `t2/reducible-query` row-by-row without materializing."- [^Writer writer reducible]- (.write writer "[")- (let [first? (volatile! true)]- (run! (fn [row]- (if @first?- (vreset! first? false)- (.write writer ","))- (json/encode-to row writer {}))- reducible))- (.write writer "]"))--(defn- write-json-object!- "Writes a JSON object whose values are JSON arrays. `entries` is a reducible of- `[entry-name reducible-rows]` pairs."- [^Writer writer entries]- (.write writer "{")- (let [first? (volatile! true)]- (run! (fn [[entry-name objects]]- (if @first? (vreset! first? false) (.write writer ","))- (.write writer (str "\"" entry-name "\":"))- (write-json-array! writer objects))- entries))- (.write writer "}"))--(mu/defn export-metadata!- "Streams the metadata export to `os` as a JSON object. Walks [[sections]] and- includes each section only when its `:enabled?` predicate is truthy for `opts`.-- Warehouses with large schemas can produce gigabytes of metadata, so rows are- pulled from reducible queries and streamed directly to the writer — memory- stays bounded regardless of schema size."- [^OutputStream os- opts :- ::schema/export-metadata-options]- (let [writer (BufferedWriter. (OutputStreamWriter. os StandardCharsets/UTF_8))]- (write-json-object!- writer- (eduction- (filter (fn [{:keys [enabled?]}] (enabled? opts)))- (map (fn [{:keys [name model]}]- [name (eduction- (map #(serdes/metadata-query-format model %))- (serdes/metadata-query model opts))]))- sections))- (.flush writer)))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/init.clj b/enterprise/backend/src/metabase_enterprise/serialization/init.cljdeleted file mode 100644index b6cd5a49416..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/init.clj+++ /dev/null@@ -1,5 +0,0 @@-(ns metabase-enterprise.serialization.init- (:require- [metabase-enterprise.serialization.metadata-file-import :as metadata-file-import]))--(metadata-file-import/init!)diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import.cljdeleted file mode 100644index 99c549d229f..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import.clj+++ /dev/null@@ -1,308 +0,0 @@-(ns metabase-enterprise.serialization.metadata-file-import- "Streams a metadata file into the appdb, populating- `:model/Database` (matched, never created), `:model/Table`, and `:model/Field`.-- Composes three building blocks:-- - [[metabase-enterprise.serialization.metadata-file-import.parsers]] streams- batches of `[line-num row]` tuples from a JSON file.- - [[metabase-enterprise.serialization.metadata-file-import.processors]] provides- the per-batch and per-depth SQL building blocks.- - [[metabase-enterprise.serialization.metadata-file-import.schemas]] defines- the per-row Malli schemas, shared with the parsers' callers.-- Strict-consistency invariants enforced by the pre-flight + depth-tagging- steps mean the merge loop has no orphan-handling code paths. The whole- merge is set-based SQL, one statement per phase per depth level."- (:require- [metabase-enterprise.serialization.metadata-file-import.parsers :as parsers]- [metabase-enterprise.serialization.metadata-file-import.processors :as processors]- [metabase.app-db.core :as mdb]- [metabase.sync.util :as sync.util]- [metabase.util.log :as log]- [metabase.warehouse-schema.models.table :as schema.table]- [toucan2.core :as t2])- (:import- (java.io File)- (java.sql Connection PreparedStatement)- (java.time Instant)- (java.util UUID)- (org.postgresql PGConnection)- (org.postgresql.copy CopyIn CopyManager)))--(set! *warn-on-reflection* true)--;;; ============================== drain + database matching ==============================--(defn- process-databases-batch!- "Per-batch handler for `:databases`. Updates `databases-by-source-id`- with `source-id → name` for every row, conjs target-db-ids onto- `matched-ids` for matches, and WARN-logs unmatched rows."- [matched-ids databases-by-source-id batch]- (reduce (fn [_ result]- (vswap! databases-by-source-id assoc (:source-id result) (:name result))- (case (:status result)- :matched- (vswap! matched-ids conj (:target-id result))-- :no-match- (log/warnf "metadata-file-import: skipped source database %s (no matching target): %s"- (pr-str (:name result)) (:detail result)))- nil)- nil- (processors/process-databases! batch)))--(defn- drain-via-jdbc-batch!- "Drive parser batches through JDBC `executeBatch` against per-staging-table- `PreparedStatement`s. The fallback path for non-Postgres app-dbs."- [^Connection conn ^File file matched-ids databases-by-source-id]- (with-open [^PreparedStatement tables-ps (.prepareStatement conn (processors/tables-insert-sql))- ^PreparedStatement fields-ps (.prepareStatement conn (processors/fields-insert-sql))]- (parsers/stream-keyed-arrays!- file processors/import-batch-size- {:databases (partial process-databases-batch! matched-ids databases-by-source-id)- :tables (fn [batch] (processors/drain-tables-batch-jdbc! tables-ps @databases-by-source-id batch))- :fields (fn [batch] (processors/drain-fields-batch-jdbc! fields-ps batch))})))--;; Only one COPY can be active per connection at a time, so we track the-;; current handle (`:tables` or `:fields`) and close it before opening the next.-(defn- drain-via-pg-copy!- "Drive parser batches through PG's COPY wire protocol via `CopyManager`,- streaming TSV-formatted bytes directly to the server."- [^Connection conn ^File file matched-ids databases-by-source-id]- (let [^PGConnection pg-conn (.unwrap conn PGConnection)- ^CopyManager copy-mgr (.getCopyAPI pg-conn)- current (volatile! {:kind nil :copy-in nil})- close-current! (fn []- (when-let [^CopyIn ci (:copy-in @current)]- (.endCopy ci)- (vreset! current {:kind nil :copy-in nil})))- ensure-copy! (fn [kind ^String sql]- (when (not= kind (:kind @current))- (close-current!)- (vreset! current {:kind kind :copy-in (.copyIn copy-mgr sql)})))]- (try- (parsers/stream-keyed-arrays!- file processors/import-batch-size- {:databases (partial process-databases-batch! matched-ids databases-by-source-id)- :tables (fn [batch]- (ensure-copy! :tables processors/tables-copy-sql)- (processors/drain-tables-batch-pg-copy! (:copy-in @current) @databases-by-source-id batch))- :fields (fn [batch]- (ensure-copy! :fields processors/fields-copy-sql)- (processors/drain-fields-batch-pg-copy! (:copy-in @current) batch))})- (close-current!)- (catch Throwable t- (try (close-current!) (catch Throwable _ nil))- (throw t)))))--;; Opens one JDBC connection with autoCommit=false and commits at the end.-;; c3p0's autoCommitOnClose=false means we must explicitly restore-;; autoCommit=true before returning the connection to the pool — otherwise the-;; next pool user inherits an open-transaction-capable connection and can leak-;; an idle-in-transaction backend.-(defn- drain-and-match-databases!- "Single-pass walk of `file`: match `:databases` against the live appdb,- drain `:tables` and `:fields` into staging. Returns the set of matched- target-db-ids."- [^File file]- (let [matched-ids (volatile! #{})- databases-by-source-id (volatile! {})- ds (mdb/data-source)]- (with-open [^Connection conn (.getConnection ds)]- (.setAutoCommit conn false)- (try- ;; PG COPY fast path; JDBC executeBatch fallback for H2/MySQL.- (case (mdb/db-type)- :postgres (drain-via-pg-copy! conn file matched-ids databases-by-source-id)- (drain-via-jdbc-batch! conn file matched-ids databases-by-source-id))- (.commit conn)- (catch Throwable t- (try (.rollback conn) (catch Throwable _ nil))- (throw t))- (finally- (try (.setAutoCommit conn true) (catch Throwable _ nil)))))- @matched-ids))--;;; ============================== Top-level orchestration ==============================--;; Writes via raw `:metabase_database` to bypass `:model/Database`'s-;; `:after-update` hook, which would schedule per-database Quartz sync triggers-;; — undesirable here, since the whole point of metadata-file-import is to skip-;; warehouse sync. App startup re-registers triggers.-(defn- mark-databases-sync-complete!- "Flip every matched target Database's `initial_sync_status` to `\"complete\"`- so the UI surfaces tables immediately."- [matched-target-db-ids]- (when (seq matched-target-db-ids)- (t2/query {:update :metabase_database- :set {:initial_sync_status "complete"}- :where [:in :id (vec matched-target-db-ids)]})))--(defn import-metadata-file!- "Run the full metadata import pipeline against the given file.-- `metadata-file` — `java.io.File` or path-string for the metadata file- (databases / tables / fields).-- Returns `:ok` on success; throws on hard-fail conditions (file not- readable, file incomplete, cycle in reference graph, mid-merge errors)."- [metadata-file]- (let [^File m-file (if (instance? File metadata-file)- metadata-file- (File. ^String metadata-file))]- ;; with-staging-tables clears the staging tables on entry and exit- ;; (try/finally), so a crashed prior attempt can't leak rows into this run.- (processors/with-staging-tables- ;; --- drain + database matching (single pass over the file) ---- (let [matched-target-db-ids (drain-and-match-databases! m-file)]- ;; --- analyze: feed the planner real selectivity for downstream UPDATEs ---- (processors/analyze-staging-tables!)- ;; --- pre-flight: every parent ref must resolve within the file ---- (processors/assert-no-orphan-refs!)- ;; --- pre-flight: NULL orphan fk-target refs (informational, not fatal) ---- (let [{:keys [count sample]} (processors/null-orphan-fk-target-refs!)]- (when (pos? count)- (log/warnf "metadata-file-import: %d orphan fk_target_field_id ref(s) NULLed (target not present in file); sample=%s"- count (pr-str sample))))- ;; --- depth tagging: assign 0..max-depth, detect cycles ---- (processors/compute-staging-depth!)- ;; --- table resolve round 1: matched-vs-insert decision for merge-tables! ---- (processors/resolve-target-table-ids-in-staging!)- ;; --- merge (one txn — every live-data write all-or-nothing) ---- (let [new-tables- (t2/with-transaction [_]- (let [insert-source-ids (processors/merge-tables!)]- ;; round 2 captures INSERT-assigned ids back into table staging- (processors/resolve-target-table-ids-in-staging!)- ;; copy target-table-ids from table staging onto field staging- (processors/resolve-target-table-ids-for-fields-in-staging!)- ;; depth-walk: per-depth resolve + UPDATE matched + INSERT new- (processors/merge-fields-by-depth!)- (mark-databases-sync-complete! matched-target-db-ids)- (processors/new-target-tables-from-staging insert-source-ids)))]- ;; Permission grants run outside the merge txn — the cluster lock- ;; shouldn't be held across the long merge work.- (doseq [[db-id rows] (group-by :db_id new-tables)]- (schema.table/set-new-tables-permissions! db-id rows)))- (log/infof "metadata-file-import: complete (matched-databases=%d)"- (count matched-target-db-ids))))- :ok))--;;; ============================== Import registry + agent ==============================-;;; In-JVM only — matches the scope of `metabase.sync.util/operation->db-ids`.-;;; Status is not durable across server restarts.--(def ^:private terminal-record-limit- "Max number of completed (`:ok`/`:error`) records retained in the registry.- In-flight (`:queued`/`:running`) records are never evicted regardless of count."- 10)--;; id -> {:id :status :file :enqueued-at :started-at :finished-at :wall-ms :error}-(defonce ^:private import-registry (atom {}))--(defonce ^:private import-agent- (agent ::serializer :error-mode :continue))--;; The id currently being run by the agent, or nil. A single-element index over-;; `import-registry` so the sync busy-check is O(1) instead of scanning.-(defonce ^:private running-import-id (atom nil))--(defn- terminal-status? [status]- (contains? #{:ok :error} status))--(defn- prune-terminal-records- "Drop the oldest terminal records past `terminal-record-limit`, ordered by- `:finished-at`. Non-terminal records are retained regardless of count."- [registry]- (let [terminal (filter (comp terminal-status? :status val) registry)]- (if (<= (count terminal) terminal-record-limit)- registry- (->> terminal- (sort-by (comp :finished-at val))- (drop-last terminal-record-limit)- (map key)- (apply dissoc registry)))))--(defn- finalize-record- "Merge terminal `attrs` onto record `id`, then prune."- [registry id attrs]- (-> registry- (update id merge attrs)- prune-terminal-records))--(defn import-status- "Registry record for import `id`, or nil if the id is unknown or has been- evicted."- [id]- (get @import-registry id))--(defn import-running?- "Truthy iff a metadata-file-import is currently in flight on this JVM."- []- (some? @running-import-id))--(defn- import-busy-reason []- (when-let [id @running-import-id]- (let [{:keys [file started-at]} (get @import-registry id)]- {:reason (format "metadata-file-import in progress (file=%s, since=%s)"- file started-at)})))--(defonce ^:private initialized? (atom false))--(defn init!- "Register hooks with the subsystems this module coordinates with (sync).- Called from `metabase-enterprise.serialization.init` at boot. Idempotent —- callers don't need to guard against multiple invocations."- []- (when (compare-and-set! initialized? false true)- (sync.util/register-busy-predicate! import-busy-reason)))--(defn- run-import*- "Agent body. Always returns `::serializer`; `:error-mode :continue` plus- the inner try/catch means a failing import never poisons the agent."- [_serializer id ^File file {:keys [delete-after?]}]- (let [path (.getAbsolutePath file)- t0 (System/nanoTime)]- (swap! import-registry update id merge {:status :running :started-at (Instant/now)})- (reset! running-import-id id)- (log/infof "metadata-file-import: starting (id=%s, file=%s)" id path)- (try- (import-metadata-file! file)- (let [wall-ms (/ (- (System/nanoTime) t0) 1e6)]- (log/infof "metadata-file-import: complete (id=%s, file=%s, wall-ms=%.0f)" id path wall-ms)- (swap! import-registry finalize-record id- {:status :ok :finished-at (Instant/now) :wall-ms wall-ms :error nil}))- (catch Throwable t- (let [wall-ms (/ (- (System/nanoTime) t0) 1e6)]- (log/errorf t "metadata-file-import: failed (id=%s, file=%s, wall-ms=%.0f)" id path wall-ms)- (swap! import-registry finalize-record id- {:status :error :finished-at (Instant/now) :wall-ms wall-ms :error (str t)})))- (finally- (reset! running-import-id nil)- (when delete-after?- (try (.delete file) (catch Throwable _ nil)))))- ::serializer))--(defn enqueue-import!- "Submit `file` to the import agent and return its import id (a string UUID).- Imports execute in arrival order; a `:queued` registry record is created- synchronously before this returns, so the caller can immediately look the id- up via `import-status`. With `{:delete-after? true}`, the agent deletes- `file` once it finishes (success or failure)."- ([^File file]- (enqueue-import! file {}))- ([^File file opts]- (let [id (str (UUID/randomUUID))- path (.getAbsolutePath file)]- (swap! import-registry assoc id {:id id- :status :queued- :file path- :enqueued-at (Instant/now)- :started-at nil- :finished-at nil- :wall-ms nil- :error nil})- (log/infof "metadata-file-import: queued (id=%s, file=%s)" id path)- (send-off import-agent run-import* id file opts)- id)))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/parsers.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/parsers.cljdeleted file mode 100644index dcf85bbce5b..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/parsers.clj+++ /dev/null@@ -1,132 +0,0 @@-(ns metabase-enterprise.serialization.metadata-file-import.parsers- "Streaming JSON parser for the metadata file importer. Walks a top-level JSON- object, advances to a named array, and emits its items in batches via a- callback. Memory bounded by the in-flight item plus the current batch buffer."- (:require- [metabase.util.performance :as perf])- (:import- (com.fasterxml.jackson.core JsonParser JsonToken)- (com.fasterxml.jackson.databind ObjectMapper)- (java.io File FileInputStream InputStreamReader Reader)- (java.nio.charset StandardCharsets)- (java.util LinkedHashMap)))--(set! *warn-on-reflection* true)--(def ^:private ^ObjectMapper object-mapper (ObjectMapper.))--(defn- advance-to-array!- "Advance `parser` from start-of-input through a top-level JSON object to the- array valued at `target-key` (parser has just consumed the START_ARRAY- token). Throws `:bad-shape` if the document doesn't begin with an object or- the value at `target-key` isn't an array; throws `:missing-key` if the key- is absent."- [^JsonParser parser ^String target-key]- (let [t (.nextToken parser)]- (when-not (= t JsonToken/START_OBJECT)- (throw (ex-info "Expected JSON document to begin with an object"- {:kind :bad-shape, :target-key target-key}))))- (loop []- (let [t (.nextToken parser)]- (cond- (or (nil? t) (= t JsonToken/END_OBJECT))- (throw (ex-info (format "Key %s not found in top-level object" (pr-str target-key))- {:kind :missing-key, :key target-key}))-- (= t JsonToken/FIELD_NAME)- (if (= target-key (.getCurrentName parser))- (let [vt (.nextToken parser)]- (when-not (= vt JsonToken/START_ARRAY)- (throw (ex-info (format "Value of %s must be an array" (pr-str target-key))- {:kind :bad-shape, :key target-key}))))- (do (.nextToken parser) (.skipChildren parser) (recur)))-- :else (recur)))))--(defn- consume-array-as-batches!- "After START_ARRAY has been consumed, walk array items into batches of- `[line-num row]` tuples (up to `batch-size`), calling `process-batch!` per- batch. Returns when END_ARRAY is consumed."- [^JsonParser parser array-key batch-size process-batch!]- (loop [batch (transient [])- line-num 0]- (let [t (.nextToken parser)]- (cond- (= t JsonToken/END_ARRAY)- (when (pos? (count batch))- (process-batch! (persistent! batch)))-- (= t JsonToken/START_OBJECT)- ;; LinkedHashMap fails (map? x), so coerce before keywordize-keys.- (let [raw (.readValueAs parser LinkedHashMap)- item (perf/keywordize-keys (into {} raw))- ln (inc line-num)- next-batch (conj! batch [ln item])]- (if (>= (count next-batch) batch-size)- (do (process-batch! (persistent! next-batch))- (recur (transient []) ln))- (recur next-batch ln)))-- (nil? t)- (throw (ex-info (format "Unexpected end-of-input in array %s" (pr-str array-key))- {:kind :bad-shape, :key array-key}))-- :else- (throw (ex-info (format "Unexpected token %s in array %s"- (some-> t .asString) (pr-str array-key))- {:kind :bad-shape, :key array-key}))))))--(defn- open-parser- ^JsonParser [^Reader reader]- (.createParser (.getFactory object-mapper) reader))--(defn stream-array-batches!- "Stream `file`'s top-level named array (string or keyword `array-key`) through- `process-batch!`. Each batch is a vector of `[line-num row]` tuples (up to- `batch-size`); `line-num` is 1-indexed and continues across batch boundaries.- Items are parsed into Clojure maps with keyword keys. Caller's responsibility- to ensure the file exists and is readable; opening errors propagate."- [^File file array-key batch-size process-batch!]- (with-open [is (FileInputStream. file)- reader (InputStreamReader. is StandardCharsets/UTF_8)- parser (open-parser reader)]- (advance-to-array! parser (name array-key))- (consume-array-as-batches! parser array-key batch-size process-batch!)))--(defn stream-keyed-arrays!- "Single-pass walk of `file`'s top-level JSON object, dispatching arrays to- per-key handlers. `handlers` is a map of keyword → fn-of-batch (each batch is- a vector of `[line-num row]` tuples, up to `batch-size`). Arrays for keys not- in `handlers` are skipped without materialization.-- Throws `:bad-shape` if the document doesn't begin with an object or if a- known key's value isn't an array. Missing keys are silently OK."- [^File file batch-size handlers]- (with-open [is (FileInputStream. file)- reader (InputStreamReader. is StandardCharsets/UTF_8)- parser (open-parser reader)]- (let [t (.nextToken parser)]- (when-not (= t JsonToken/START_OBJECT)- (throw (ex-info "Expected JSON document to begin with an object"- {:kind :bad-shape}))))- (loop []- (let [t (.nextToken parser)]- (cond- (or (nil? t) (= t JsonToken/END_OBJECT))- nil-- (= t JsonToken/FIELD_NAME)- (let [field-name (.getCurrentName parser)- handler-fn (get handlers (keyword field-name))]- (if handler-fn- (let [vt (.nextToken parser)]- (when-not (= vt JsonToken/START_ARRAY)- (throw (ex-info (format "Value of %s must be an array" (pr-str field-name))- {:kind :bad-shape, :key field-name})))- (consume-array-as-batches! parser field-name batch-size handler-fn)- (recur))- (do (.nextToken parser)- (.skipChildren parser)- (recur))))-- :else (recur))))))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/processors.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/processors.cljdeleted file mode 100644index 626f4c17847..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/processors.clj+++ /dev/null@@ -1,838 +0,0 @@-(ns metabase-enterprise.serialization.metadata-file-import.processors- "SQL building blocks for the metadata file importer's drain-then-merge flow.-- Errors propagate as `ex-info` with `:kind` for typed failure handling- (`:invalid-input`, `:file-incomplete`, `:cycle-in-field-graph`, `:depth-tagging-cap-exceeded`)."- (:require- [malli.error :as me]- [metabase-enterprise.serialization.metadata-file-import.schemas :as schemas]- [metabase.app-db.core :as mdb]- [metabase.models.humanization :as humanization]- [metabase.util.json :as json]- [metabase.util.malli.registry :as mr]- [toucan2.core :as t2])- (:import- (java.nio.charset StandardCharsets)- (java.sql PreparedStatement Types)- (org.postgresql.copy CopyIn)))--(set! *warn-on-reflection* true)--(def import-batch-size- "Row batch size shared by the drain and merge processors."- ;; 250 was the sweet spot in local benchmarks- 250)--;;; ============================== Validation ==============================--(defn- validate-line!- "Validate `line` against the registered Malli `schema-ref`.- On failure throws `ex-info` with `:kind :invalid-input`, `:line`, a humanized `:detail`,- and `extras` merged into the ex-data."- [schema-ref line-num line extras]- (when-let [humanized (me/humanize (mr/explain schema-ref line))]- (throw (ex-info (format "Invalid %s row on line %d" (name schema-ref) line-num)- (merge extras- {:kind :invalid-input- :line line-num- :detail (pr-str humanized)})))))--;;; ============================== Staging tables ==============================--(defn clear-staging-tables!- "Empty `metabase_table_import` and `metabase_field_import`."- []- ;; TRUNCATE (not DELETE) so multiple imports in one process lifetime don't- ;; accumulate dead tuples in staging.- ;; Called outside any outer transaction, so MySQL's implicit-commit-on-- ;; TRUNCATE doesn't break composition.- (t2/query {:truncate :metabase_table_import})- (t2/query {:truncate :metabase_field_import}))--(defn analyze-staging-tables!- "Refresh planner statistics on `metabase_table_import` and `metabase_field_import`."- []- (case (mdb/db-type)- :postgres- (do (t2/query "ANALYZE metabase_table_import")- (t2/query "ANALYZE metabase_field_import"))- :mysql- (do (t2/query "ANALYZE TABLE metabase_table_import")- (t2/query "ANALYZE TABLE metabase_field_import"))- :h2- (t2/query "ANALYZE")))--(defmacro with-staging-tables- "Run `body` with staging tables cleared on entry and on exit."- [& body]- `(do (clear-staging-tables!)- (try ~@body- (finally (clear-staging-tables!)))))--;;; ============================== databases (batch) ==============================--(defn- engine-name- "Normalize `engine` (string or keyword) to a string for natural-key comparison.- Toucan2's `:model/Database` reads `:engine` as a keyword via `define-after-select`,- but file-imported rows always carry strings. Normalize both sides to string."- [engine]- (when engine (name engine)))--(defn- match-databases-batch- "Look up every existing target Database matching any source row in the- batch. Returns `{[name engine-string] → existing-id}` for matching rows."- [lines]- (let [pairs (into #{}- (map (fn [{:keys [name engine]}] [name (engine-name engine)]))- lines)- names (into #{} (map first) pairs)- engines (into #{} (map second) pairs)]- (if (or (empty? names) (empty? engines))- {}- ;; Over-include via `(name IN ..., engine IN ...)` then intersect in- ;; Clojure — keeps the SQL portable (no `(col, col) IN ((?, ?), ...)`- ;; tuple form across Postgres / H2 / MySQL).- (let [rows (t2/select [:model/Database :id :name :engine]- {:where [:and- [:in :name names]- [:in :engine engines]]})]- (into {}- (comp (map (fn [{:keys [id name engine]}]- [[name (engine-name engine)] id]))- (filter (fn [[pair _]] (contains? pairs pair))))- rows)))))--(defn process-databases!- "Process a batch of database rows. Returns an eduction of result maps.-- Result shapes:- `{:source-id <int> :name <string> :target-id <int> :status :matched}`- `{:source-id <int> :name <string> :status :no-match :line L :detail S}`-- Validation failures throw; lookup misses produce `:no-match` results- (non-fatal)."- [batch]- (doseq [[ln line] batch]- (validate-line! ::schemas/database-info ln line {:source-id (:id line)}))- (let [match-idx (match-databases-batch (mapv second batch))]- (eduction- (map (fn [[ln {:keys [id name engine]}]]- (if-let [target (match-idx [name (engine-name engine)])]- {:source-id id :name name :target-id target :status :matched}- {:source-id id :name name- :status :no-match- :line ln- :detail (format "No database with name=%s engine=%s"- (pr-str name) (pr-str (engine-name engine)))})))- batch)))--;;; ============================== tables — drain + merge ==============================--(defn- encode-path-or-nil- "Encode an `:nfc_path` coll as a JSON string. NULL for empty/nil."- [coll]- (when (seq coll) (json/encode (vec coll))))--;;; ============================== JDBC drain (perf path) ==============================-;;;-;;; Hand-rolled JDBC `executeBatch` against single per-staging-table prepared-;;; statements. Used by the loader's `drain-and-match-databases!` to bypass-;;; the t2/insert! + HoneySQL VALUES formatting overhead — measured ~15×-;;; slower per row than PG's COPY floor on multi-million-row drains.--(defn tables-insert-sql- "Parameterized INSERT for `metabase_table_import`. Column order matches the- `set*-batch!` parameter-binding order in [[drain-tables-batch-jdbc!]]."- []- (str "INSERT INTO metabase_table_import"- ;; `schema` is a reserved word in MySQL/MariaDB- " (source_id, source_db_id, db_name, " (mdb/quote-for-application-db "schema")- ", name, description, display_name)"- " VALUES (?, ?, ?, ?, ?, ?, ?)"))--(defn fields-insert-sql- "Parameterized INSERT for `metabase_field_import`. Column order matches the- `set*-batch!` parameter-binding order in [[drain-fields-batch-jdbc!]]."- []- (str "INSERT INTO metabase_field_import"- " (source_id, source_table_id, source_parent_id, source_fk_target_id,"- " name, base_type, database_type, effective_type, semantic_type,"- " coercion_strategy, description, nfc_path)"- " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"))--(defn- set-int-or-null! [^PreparedStatement ps ^long idx v]- (if v- (.setInt ps idx (int v))- (.setNull ps idx Types/INTEGER)))--(defn- set-string-or-null! [^PreparedStatement ps ^long idx v]- (if v- (.setString ps idx (str v))- (.setNull ps idx Types/VARCHAR)))--(defn drain-tables-batch-jdbc!- "Per-batch handler for `:tables` that binds rows onto `ps` and flushes via- `executeBatch`. Validates each row first; throws on schema failure.-- `ps` must be a `PreparedStatement` for [[tables-insert-sql]]; the caller- owns the connection + statement lifecycle."- [^PreparedStatement ps databases-by-source-id batch]- (doseq [[ln line] batch]- (validate-line! ::schemas/table-info ln line {:source-id (:id line)}))- (when (seq batch)- (doseq [[_ {:keys [id db_id schema name description]}] batch]- (.setInt ps 1 (int id))- (.setInt ps 2 (int db_id))- (.setString ps 3 (str (get databases-by-source-id db_id)))- (set-string-or-null! ps 4 schema)- (.setString ps 5 (str name))- (set-string-or-null! ps 6 description)- (.setString ps 7 (humanization/name->human-readable-name name))- (.addBatch ps))- (.executeBatch ps)))--(defn drain-fields-batch-jdbc!- "Per-batch handler for `:fields` that binds rows onto `ps` and flushes via- `executeBatch`. Validates each row first; throws on schema failure.-- `ps` must be a `PreparedStatement` for [[fields-insert-sql]]; the caller- owns the connection + statement lifecycle."- [^PreparedStatement ps batch]- (doseq [[ln line] batch]- (validate-line! ::schemas/field-info ln line {:source-id (:id line)}))- (when (seq batch)- (doseq [[_ {:keys [id table_id parent_id fk_target_field_id- name base_type database_type- effective_type semantic_type- coercion_strategy description nfc_path]}] batch]- (.setInt ps 1 (int id))- (.setInt ps 2 (int table_id))- (set-int-or-null! ps 3 parent_id)- (set-int-or-null! ps 4 fk_target_field_id)- (.setString ps 5 (str name))- (.setString ps 6 (str base_type))- (.setString ps 7 (str database_type))- (set-string-or-null! ps 8 effective_type)- (set-string-or-null! ps 9 semantic_type)- (set-string-or-null! ps 10 coercion_strategy)- (set-string-or-null! ps 11 description)- (set-string-or-null! ps 12 (encode-path-or-nil nfc_path))- (.addBatch ps))- (.executeBatch ps)))--;;; ============================== PG-specific COPY drain ==============================-;;;-;;; Drives staging-table inserts through PostgreSQL's COPY wire protocol via-;;; `CopyManager` / `CopyIn`. Streams TSV-formatted row bytes directly to the-;;; server — no per-statement parsing, no JDBC parameter binding overhead.-;;; Approaches PG's raw bulk-load throughput; in practice ~5× faster than the-;;; JDBC `executeBatch` path at multi-million-row scale.--(def tables-copy-sql- "PostgreSQL COPY statement for `metabase_table_import`. Column order matches- the TSV emitted by [[table-tsv-line]]."- (str "COPY metabase_table_import"- " (source_id, source_db_id, db_name, schema, name, description, display_name)"- " FROM STDIN"))--(def fields-copy-sql- "PostgreSQL COPY statement for `metabase_field_import`. Column order matches- the TSV emitted by [[field-tsv-line]]."- (str "COPY metabase_field_import"- " (source_id, source_table_id, source_parent_id, source_fk_target_id,"- " name, base_type, database_type, effective_type, semantic_type,"- " coercion_strategy, description, nfc_path)"- " FROM STDIN"))--(defn- tsv-escape ^String [^String s]- (-> s- (.replace "\\" "\\\\")- (.replace "\t" "\\t")- (.replace "\n" "\\n")- (.replace "\r" "\\r")))--(defn- tsv-cell ^String [v]- (cond- (nil? v) "\\N"- (number? v) (str v)- (boolean? v) (str v)- :else (tsv-escape (str v))))--(defn- table-tsv-line ^String [row databases-by-source-id]- (let [{:keys [id db_id schema name description]} row]- (str (tsv-cell id) "\t"- (tsv-cell db_id) "\t"- (tsv-cell (get databases-by-source-id db_id)) "\t"- (tsv-cell schema) "\t"- (tsv-cell name) "\t"- (tsv-cell description) "\t"- (tsv-cell (humanization/name->human-readable-name name)) "\n")))--(defn- field-tsv-line ^String [row]- (let [{:keys [id table_id parent_id fk_target_field_id- name base_type database_type- effective_type semantic_type- coercion_strategy description nfc_path]} row]- (str (tsv-cell id) "\t"- (tsv-cell table_id) "\t"- (tsv-cell parent_id) "\t"- (tsv-cell fk_target_field_id) "\t"- (tsv-cell name) "\t"- (tsv-cell base_type) "\t"- (tsv-cell database_type) "\t"- (tsv-cell effective_type) "\t"- (tsv-cell semantic_type) "\t"- (tsv-cell coercion_strategy) "\t"- (tsv-cell description) "\t"- (tsv-cell (encode-path-or-nil nfc_path)) "\n")))--(defn drain-tables-batch-pg-copy!- "Per-batch handler for `:tables` that streams TSV rows over PG's COPY wire- protocol via the supplied `CopyIn` handle. Validates each row first."- [^CopyIn copy-in databases-by-source-id batch]- (doseq [[ln line] batch]- (validate-line! ::schemas/table-info ln line {:source-id (:id line)}))- (doseq [[_ row] batch]- (let [^bytes bs (.getBytes (table-tsv-line row databases-by-source-id) StandardCharsets/UTF_8)]- (.writeToCopy copy-in bs 0 (alength bs)))))--(defn drain-fields-batch-pg-copy!- "Per-batch handler for `:fields` that streams TSV rows over PG's COPY wire- protocol via the supplied `CopyIn` handle. Validates each row first."- [^CopyIn copy-in batch]- (doseq [[ln line] batch]- (validate-line! ::schemas/field-info ln line {:source-id (:id line)}))- (doseq [[_ row] batch]- (let [^bytes bs (.getBytes (field-tsv-line row) StandardCharsets/UTF_8)]- (.writeToCopy copy-in bs 0 (alength bs)))))--(defn resolve-target-table-ids-in-staging!- "Set `metabase_table_import.target_id` to the int id of the matching- `metabase_table` row for every staging row whose match key resolves. The- match key is `(db_name, schema, name)` against `(d.name, t.schema, t.name)`,- restricted to non-defective live rows.-- Inactive matches will be reactivated on import.-- Rows that do not match keep `target_id` NULL. Idempotent — running again with- the same staging contents produces the same assignments."- []- (t2/query- {:update :metabase_table_import- :set {:target_id- ;; MIN(t.id): `metabase_database`'s unique constraint is- ;; `(router_database_id, name)`, so two NULL-router databases- ;; can share a name → two `(db_name, schema, name)` matches- ;; (GHY-3549).- {:select [[[:min :t.id]]]- :from [[:metabase_table :t]]- :join [[:metabase_database :d] [:= :d.id :t.db_id]]- :where [:and- [:= :metabase_table_import.db_name :d.name]- [:= [:coalesce :t.schema [:inline ""]]- [:coalesce :metabase_table_import.schema [:inline ""]]]- [:= :t.name :metabase_table_import.name]- [:= :t.is_defective_duplicate [:inline false]]]}}}))--;; UPDATE before INSERT: on a clean-schema import the UPDATE matches nothing-;; (fast no-op) and the INSERT then writes each row exactly once.-;;-;; Callers re-run [[resolve-target-table-ids-in-staging!]] and pass the set to-;; [[new-target-tables-from-staging]] to read back the freshly-created rows.-(defn merge-tables!- "Merge `metabase_table_import` into `metabase_table` atomically: UPDATE the- staging rows that matched a live table, INSERT the rest. Returns the set of- staging `source_id`s that were INSERTed (snapshot taken before the INSERT).-- Pre-condition: [[resolve-target-table-ids-in-staging!]] must have run, so- matched staging rows carry a `target_id`."- []- (t2/with-transaction [_]- ;; t2 joins an outer txn rather than creating a savepoint, so wrapping- ;; this call in a larger transaction is safe.-- ;; Capture which staging rows are about to be INSERTed (target_id NULL).- ;; After the INSERT + a subsequent resolve round, these source_ids will- ;; have their `target_id` populated with the freshly-inserted live id.- (let [insert-source-ids- (into #{} (map :source_id)- (t2/query {:select [:source_id]- :from [:metabase_table_import]- :where [:= :target_id nil]}))]- ;; UPDATE matched rows first (clobber description, bump updated_at).- ;; Skip-if-unchanged: the EXISTS subquery additionally requires that- ;; description actually differs (NULL-safe via COALESCE-with-empty-- ;; string), so a re-import with identical description is a true no-op- ;; — no UPDATE fires, no dead tuple. updated_at is deliberately NOT- ;; in the predicate (otherwise the UPDATE would always fire,- ;; defeating the optimization).- (t2/query- {:update :metabase_table- :set {:description {;; ORDER BY + LIMIT 1: two staging rows can- ;; resolve to the same `target_id` (GHY-3549).- :select [:it.description]- :from [[:metabase_table_import :it]]- :where [:= :it.target_id :metabase_table.id]- :order-by [[:it.source_id :asc]]- :limit 1}- ;; reactivate in place:- :active [:inline true]- :updated_at :%now}- :where [:and- [:= :metabase_table.is_defective_duplicate [:inline false]]- [:exists {:select [[[:inline 1]]]- :from [[:metabase_table_import :it]]- :where [:and- [:= :it.target_id :metabase_table.id]- ;; Fire when the description differs OR the row needs- ;; reactivating — otherwise re-importing a deactivated- ;; but otherwise-unchanged table would skip the UPDATE- ;; and leave it inactive.- [:or- [:!= [:coalesce :metabase_table.description [:inline ""]]- [:coalesce :it.description [:inline ""]]]- [:= :metabase_table.active [:inline false]]]]}]]})- ;; INSERT bypasses :model/Table's :after-insert hook — that hook- ;; schedules per-DB Quartz triggers we don't want and fires- ;; `set-new-table-permissions!` once per row. The JOIN on db_name- ;; silently drops staging rows whose source DB has no target appdb.- (t2/query- {:insert-into- [[:metabase_table [:db_id :schema :name :description :display_name :data_layer- :active :show_in_getting_started :is_defective_duplicate- :created_at :updated_at]]- {:select [:d.id :it.schema :it.name :it.description :it.display_name- [[:inline "internal"]]- [[:inline true]] [[:inline false]] [[:inline false]]- :%now :%now]- :from [[:metabase_table_import :it]]- :join [[:metabase_database :d] [:= :d.name :it.db_name]]- :where [:= :it.target_id nil]}]})- insert-source-ids)))--(defn new-target-tables-from-staging- "Look up the `metabase_table` rows that were just INSERTed by- [[merge-tables!]]. `insert-source-ids` is the set returned by- [[merge-tables!]]. Pre-condition:- [[resolve-target-table-ids-in-staging!]] has run since the INSERT, so- staging's `target_id` is populated for these rows."- [insert-source-ids]- (when (seq insert-source-ids)- (t2/query {:select [:t.id :t.db_id :t.schema]- :from [[:metabase_table :t]]- :join [[:metabase_table_import :it] [:= :it.target_id :t.id]]- :where [:in :it.source_id (vec insert-source-ids)]})))--;;; ============================== Pre-flight orphan check ==============================--(def ^:private orphan-sample-cap- "How many orphan rows to surface in the error data when bailing out. Visibility- without exploding the exception payload."- 10)--(defn- orphan-not-exists-predicate- "WHERE-clause fragment matching `metabase_field_import` rows whose `column-key` (a `:source_*_id` column)- is non-null but references no other staging row's `source_id` — i.e. an orphan reference."- [column-key]- ;; Correlated NOT EXISTS, not NOT IN: at 9M+ rows PG can't plan- ;; `NOT IN (subquery)` (NULL-semantics force full-materialization of the- ;; inner set) and it effectively never terminates. NOT EXISTS resolves to a- ;; proper anti-join via the PK index on `source_id` — O(N log N), not O(N²).- ;;- ;; The inner FROM is a `(SELECT source_id FROM ...)` derived table so the- ;; predicate stays valid inside an UPDATE on `metabase_field_import`: MySQL- ;; forbids referencing the UPDATE's target table directly in a subquery; the- ;; wrap defeats that, and PG/H2 inline it away. Aliased `:s` so the inner- ;; WHERE's unqualified column-key resolves to the outer table.- (let [outer-col (keyword (str "metabase_field_import." (name column-key)))]- [:and- [:not= column-key nil]- [:not [:exists- {:select [[[:inline 1]]]- :from [[{:select [:source_id]- :from [:metabase_field_import]} :s]]- :where [:= :s.source_id outer-col]}]]]))--(defn- orphan-count- "Number of `metabase_field_import` rows whose `column-key` (a `:source_*_id`- column) is non-null but doesn't reference any other row's `source_id`."- [column-key]- (t2/count :metabase_field_import {:where (orphan-not-exists-predicate column-key)}))--(defn- orphan-sample- "Up to `orphan-sample-cap` `[source_id, <column-key>]` pairs for orphan rows."- [column-key]- (t2/query- {:select [:source_id column-key]- :from [:metabase_field_import]- :where (orphan-not-exists-predicate column-key)- :limit orphan-sample-cap}))--(defn assert-no-orphan-refs!- "Pre-flight check after drain: every staging field row's `source_parent_id`- must reference another staging row's `source_id`. Throws `ex-info` with- `:kind :file-incomplete` and a sample of orphan rows in the error data- when any orphan parent ref exists.-- Parent refs are structural — a field claiming to be a child of a missing- field can't be positioned, so the file is genuinely incomplete. Orphan- `source_fk_target_id` refs are not fatal here; see- [[null-orphan-fk-target-refs!]]."- []- (let [parent-count (orphan-count :source_parent_id)]- (when (pos? parent-count)- (throw (ex-info (format "metadata-file-import: file is incomplete — %d orphan parent ref(s)"- parent-count)- {:kind :file-incomplete- :orphan-parent-count parent-count- :orphan-parent-sample (orphan-sample :source_parent_id)})))))--(defn null-orphan-fk-target-refs!- "Find staging rows whose `source_fk_target_id` points at a `source_id`- not present in staging, and NULL the `source_fk_target_id` column on- those rows. Returns `{:count N :sample [...]}` when at least one row was- scrubbed, `{:count 0}` otherwise."- []- ;; Why NULL instead of abort: fk_target refs that cross a hidden/archived- ;; table boundary on the source side (the table is `active=false` or- ;; `visibility_type='hidden'`) survive in `metabase_field` but get filtered- ;; out by the export's table-visibility join. The exported file then contains- ;; fk_target refs whose targets aren't emitted — a real-data shape we observe- ;; in production appdbs. Treating these as fatal would block legitimate- ;; imports; treating them as no-fk is the lossless choice for the importer.- (let [n (orphan-count :source_fk_target_id)]- (if (zero? n)- {:count 0}- (let [sample (orphan-sample :source_fk_target_id)]- (t2/query- {:update :metabase_field_import- :set {:source_fk_target_id nil}- :where (orphan-not-exists-predicate :source_fk_target_id)})- {:count n :sample sample}))))--;;; ============================== Depth tagging ==============================--(def depth-iteration-cap- "Iteration cap on the depth-tagging fixpoint loop; exceeding it throws- `:cycle-in-field-graph` with the un-tagged sample."- ;; Field parent chains are shallow in practice (a few levels at most), so 50- ;; is a comfortable safety belt — exceeding it means the algorithm is the- ;; bug, not the data.- 50)--(defn- mark-roots-at-depth-zero!- "Tag every staging row with no parent and no fk_target ref as `depth = 0`.- Bootstraps the iteration."- []- (t2/query- {:update :metabase_field_import- :set {:depth 0}- :where [:and- [:= :source_parent_id nil]- [:= :source_fk_target_id nil]]}))--(defn- mark-rows-at-depth!- "Tag every still-untagged staging row whose `source_parent_id` and- `source_fk_target_id` (when non-NULL) reference rows that already have- `depth < d`."- [d]- ;; MySQL forbids referencing the UPDATE's target table directly inside a- ;; subquery; wrap in a derived table there. PG/H2 accept the direct form.- (let [staging-source (case (mdb/db-type)- :mysql {:select [:source_id :depth]- :from [:metabase_field_import]}- :metabase_field_import)]- (t2/query- {:update :metabase_field_import- :set {:depth d}- :where [:and- [:= :metabase_field_import.depth nil]- [:or- [:= :metabase_field_import.source_parent_id nil]- [:exists {:select [[[:inline 1]]]- :from [[staging-source :p]]- :where [:and- [:= :p.source_id :metabase_field_import.source_parent_id]- [:not= :p.depth nil]- [:< :p.depth d]]}]]- [:or- [:= :metabase_field_import.source_fk_target_id nil]- [:exists {:select [[[:inline 1]]]- :from [[staging-source :f]]- :where [:and- [:= :f.source_id :metabase_field_import.source_fk_target_id]- [:not= :f.depth nil]- [:< :f.depth d]]}]]]})))--(defn- untagged-staging-row-count- "Number of `metabase_field_import` rows still at `depth IS NULL`. Used as- the convergence signal in `compute-staging-depth!`."- []- (t2/count :metabase_field_import :depth nil))--(defn- untagged-staging-row-sample- "Up to `orphan-sample-cap` un-tagged rows for cycle-error diagnostics."- []- (t2/query- {:select [:source_id :source_parent_id :source_fk_target_id]- :from [:metabase_field_import]- :where [:= :depth nil]- :limit orphan-sample-cap}))--;; The fixpoint loop terminates on convergence (no untagged rows) or a-;; no-progress round. The `assert-no-orphan-refs!` pre-condition is what lets-;; us read no-progress as a cycle: with orphans already ruled out, a row that-;; can never be tagged must be part of one.-(defn compute-staging-depth!- "Tag every `metabase_field_import` row with a non-NULL `depth`: 0 for roots- (no parent, no fk_target ref), `d` for rows whose deps are all at depth- < `d`. Returns the maximum depth in staging.-- Pre-condition: [[assert-no-orphan-refs!]] must have run successfully.-- Throws `:cycle-in-field-graph` if rows can't be tagged (a reference cycle)- or `:depth-tagging-cap-exceeded` if the loop runs past [[depth-iteration-cap]]."- []- (mark-roots-at-depth-zero!)- (loop [d 1]- (let [before (untagged-staging-row-count)]- (cond- (zero? before)- nil-- (>= d depth-iteration-cap)- (throw (ex-info (format "metadata-file-import: depth-tagging exceeded cap of %d iterations" depth-iteration-cap)- {:kind :depth-tagging-cap-exceeded- :iterations d- :remaining-rows-count before- :remaining-rows-sample (untagged-staging-row-sample)}))-- :else- (do- (mark-rows-at-depth! d)- (let [after (untagged-staging-row-count)]- (if (= before after)- (throw (ex-info (format "metadata-file-import: %d staging row(s) could not be tagged with depth — cycle in file's parent/fk_target reference graph"- after)- {:kind :cycle-in-field-graph- :remaining-rows-count after- :iterations d- :remaining-rows-sample (untagged-staging-row-sample)}))- (recur (inc d))))))))- (or (:max (first (t2/query {:select [[[:max :depth] :max]]- :from [:metabase_field_import]})))- 0))--;;; ============================== Per-depth field merge ==============================--(def ^:private field-clobber-cols- "The field payload columns the importer owns: on a matched row they are- overwritten from the file's values (the import is an alternate sync)."- ;; Each is also its staging column name, so [[update-matched-fields-at-depth!]]- ;; can build the SET from identically-named correlated subqueries.- [:base_type :database_type :description- :effective_type :semantic_type :coercion_strategy :nfc_path])--(defn- target-id-clobber-subquery- "Correlated subquery yielding `staging-col`'s value from the staging row- that resolved to this `metabase_field`."- [staging-col]- {:select [(keyword (str "fi." (name staging-col)))]- :from [[:metabase_field_import :fi]]- :where [:= :fi.target_id :metabase_field.id]- ;; ORDER BY + LIMIT 1: two staging rows can resolve to the same target_id- ;; (GHY-3549); pick one deterministically rather than erroring.- :order-by [[:fi.source_id :asc]]- :limit 1})--(def ^:private field-payload-changed-predicate- "True when a matched `metabase_field` row's observable payload differs from- its staging row — the predicate behind the merge UPDATE's skip-if-unchanged- behavior."- ;; Coalesce-with-sentinel is the portable NULL-safe `IS DISTINCT FROM`:- ;; empty string for text columns, -1 (never a real id) for FK ids.- [[:!= [:coalesce :metabase_field.base_type [:inline ""]] [:coalesce :fi.base_type [:inline ""]]]- [:!= [:coalesce :metabase_field.database_type [:inline ""]] [:coalesce :fi.database_type [:inline ""]]]- [:!= [:coalesce :metabase_field.description [:inline ""]] [:coalesce :fi.description [:inline ""]]]- ;; `effective_type` coalesces against `base_type`: the export omits it when- ;; the two are equal, and the app reads NULL effective_type as base_type.- [:!= [:coalesce :metabase_field.effective_type :metabase_field.base_type]- [:coalesce :fi.effective_type :fi.base_type]]- [:!= [:coalesce :metabase_field.semantic_type [:inline ""]] [:coalesce :fi.semantic_type [:inline ""]]]- [:!= [:coalesce :metabase_field.coercion_strategy [:inline ""]] [:coalesce :fi.coercion_strategy [:inline ""]]]- [:!= [:coalesce :metabase_field.nfc_path [:inline ""]] [:coalesce :fi.nfc_path [:inline ""]]]- ;; staging's FK column is `target_fk_target_id`, not `fk_target_field_id`.- [:!= [:coalesce :metabase_field.fk_target_field_id [:inline -1]] [:coalesce :fi.target_fk_target_id [:inline -1]]]- ;; `active` compares against TRUE — the merge SET always sets it TRUE.- [:!= :metabase_field.active [:inline true]]])--(defn resolve-target-table-ids-for-fields-in-staging!- "Populate `metabase_field_import.target_table_id` by joining through- `metabase_table_import.source_id → metabase_table_import.target_id`.-- Pre-condition: `metabase_table_import.target_id` must be populated- (via [[resolve-target-table-ids-in-staging!]]) for both matched and- newly-inserted target rows.-- Field rows whose source table has no target (orphan-source-database- case) keep `target_table_id NULL`."- []- (t2/query- {:update :metabase_field_import- :set {:target_table_id- {:select [:ti.target_id]- :from [[:metabase_table_import :ti]]- :where [:= :ti.source_id :metabase_field_import.source_table_id]}}}))--(defn fill-target-parent-ids-at-depth!- "Populate `target_parent_id` for depth-`d` staging rows by self-joining- staging on `source_parent_id → source_id` and reading the parent's- `target_id`. Pre-condition: all prior-depth staging rows have `target_id`- resolved (the depth walk processes lower depths first, so this is- guaranteed when called in order)."- [d]- ;; Same MySQL same-table-in-UPDATE workaround as [[mark-rows-at-depth!]].- (let [parent-source (case (mdb/db-type)- :mysql {:select [:source_id :target_id]- :from [:metabase_field_import]}- :metabase_field_import)]- (t2/query- {:update :metabase_field_import- :set {:target_parent_id- {:select [:p.target_id]- :from [[parent-source :p]]- :where [:= :p.source_id :metabase_field_import.source_parent_id]}}- :where [:and- [:= :metabase_field_import.depth d]- [:not= :metabase_field_import.source_parent_id nil]]})))--(defn fill-target-fk-target-ids-at-depth!- "Populate `target_fk_target_id` for depth-`d` staging rows by self-joining- staging on `source_fk_target_id → source_id`."- [d]- ;; Same MySQL same-table-in-UPDATE workaround as [[mark-rows-at-depth!]].- (let [fk-source (case (mdb/db-type)- :mysql {:select [:source_id :target_id]- :from [:metabase_field_import]}- :metabase_field_import)]- (t2/query- {:update :metabase_field_import- :set {:target_fk_target_id- {:select [:f.target_id]- :from [[fk-source :f]]- :where [:= :f.source_id :metabase_field_import.source_fk_target_id]}}- :where [:and- [:= :metabase_field_import.depth d]- [:not= :metabase_field_import.source_fk_target_id nil]]})))--(defn resolve-target-field-ids-at-depth!- "Set `target_id` for depth-`d` staging rows by natural-key match against `metabase_field`. Matches- by `(target_table_id, name, target_parent_id)` with NULL-safe parent comparison (a staging row with- no parent matches only against `metabase_field.parent_id IS NULL`). Skips defective duplicates on- the target side.-- Run twice per depth — once before the UPDATE/INSERT step (to identify matches for the clobber path)- and once after (to capture INSERT ids so higher-depth rows can find them as parents/FK targets)."- [d]- (t2/query- {:update :metabase_field_import- :set {:target_id- ;; MIN(f.id) tiebreaker — `metabase_field` has no uniqueness- ;; constraint on `(table_id, name, parent_id)` (GHY-3549).- {:select [[[:min :f.id]]]- :from [[:metabase_field :f]]- :where [:and- [:= :f.table_id :metabase_field_import.target_table_id]- [:= :f.name :metabase_field_import.name]- [:or- [:and [:= :metabase_field_import.target_parent_id nil]- [:= :f.parent_id nil]]- [:= :f.parent_id :metabase_field_import.target_parent_id]]- [:= :f.is_defective_duplicate [:inline false]]]}}- :where [:and- [:= :metabase_field_import.depth d]- [:= :metabase_field_import.target_id nil]]}))--(defn update-matched-fields-at-depth!- "Bring the live `metabase_field` rows that depth-`d` staging matched into line with the file. Rows- whose payload already matches staging are left untouched."- [d]- (t2/query- {:update :metabase_field- ;; `fk_target_field_id` reads staging's `target_fk_target_id` — it isn't- ;; part of the natural-key match, so a matched field's FK can drift.- :set (-> (into {} (map (fn [c] [c (target-id-clobber-subquery c)])) field-clobber-cols)- (assoc :fk_target_field_id (target-id-clobber-subquery :target_fk_target_id)- :active [:inline true]- :updated_at :%now))- :where [:and- [:= :metabase_field.is_defective_duplicate [:inline false]]- ;; Scope the outer walk to the ~staging-row-count IDs at depth `d`.- ;; Without this, PG starts from `metabase_field` and probes staging- ;; per row — a full scan on an appdb with millions of field rows.- [:in :metabase_field.id {:select [:target_id]- :from [:metabase_field_import]- :where [:and- [:= :depth d]- [:not= :target_id nil]]}]- ;; Skip-if-unchanged: only touch rows whose payload actually differs.- [:exists {:select [[[:inline 1]]]- :from [[:metabase_field_import :fi]]- :where [:and- [:= :fi.target_id :metabase_field.id]- [:= :fi.depth d]- (into [:or] field-payload-changed-predicate)]}]]}))--(defn insert-new-fields-at-depth!- "INSERT `metabase_field` for depth-`d` staging rows that didn't match a- live row (`target_id IS NULL`). The new row's `parent_id` and- `fk_target_field_id` come from staging's `target_parent_id` /- `target_fk_target_id` — populated earlier in this depth's iteration.-- Rows whose `target_table_id IS NULL` (source table didn't match target)- are silently dropped. Rows whose `source_parent_id IS NOT NULL` but- `target_parent_id IS NULL` are also dropped — that's the case where the- parent field's table didn't match target, cascading the orphan downward."- [d]- (t2/query- {:insert-into- [[:metabase_field [:table_id :name :base_type :database_type :description- :effective_type :semantic_type :coercion_strategy- :nfc_path :parent_id :fk_target_field_id- :is_defective_duplicate :active :created_at :updated_at]]- {:select [:fi.target_table_id :fi.name :fi.base_type :fi.database_type :fi.description- :fi.effective_type :fi.semantic_type :fi.coercion_strategy- :fi.nfc_path :fi.target_parent_id :fi.target_fk_target_id- [[:inline false]] [[:inline true]] :%now :%now]- :from [[:metabase_field_import :fi]]- :where [:and- [:= :fi.depth d]- [:= :fi.target_id nil]- [:not= :fi.target_table_id nil]- [:or- [:= :fi.source_parent_id nil]- [:not= :fi.target_parent_id nil]]]}]}))--(defn merge-fields-by-depth!- "Merge `metabase_field_import` into `metabase_field`, walking depth 0 to- max-depth so a row's parents and FK targets are resolved before the row- itself. All depths run inside one transaction.-- Pre-conditions: [[compute-staging-depth!]] has populated `depth` and- [[resolve-target-table-ids-for-fields-in-staging!]] has populated- `target_table_id`."- []- (let [max-d (or (:max (first (t2/query {:select [[[:max :depth] :max]]- :from [:metabase_field_import]})))- 0)]- ;; One transaction across all depths — t2 joins the import's outer txn.- (t2/with-transaction [_]- (doseq [d (range (inc max-d))]- (fill-target-parent-ids-at-depth! d)- (fill-target-fk-target-ids-at-depth! d)- (resolve-target-field-ids-at-depth! d)- (update-matched-fields-at-depth! d)- (insert-new-fields-at-depth! d)- ;; Re-resolve to capture INSERT ids, so higher depths can find these- ;; rows as parents / FK targets.- (resolve-target-field-ids-at-depth! d)))))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/schemas.clj b/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/schemas.cljdeleted file mode 100644index 6b5ab8968ae..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/metadata_file_import/schemas.clj+++ /dev/null@@ -1,64 +0,0 @@-(ns metabase-enterprise.serialization.metadata-file-import.schemas- "Malli schemas describing the wire-format rows streamed by- `GET /api/ee/serialization/metadata/export` and consumed by- `POST /api/ee/serialization/metadata/import`.-- IDs are **integers from the source appdb**. They are internally consistent- within one file (so `:fields[].parent_id` references a `:fields[].id` in the- same file) but mean nothing across files.-- Cross-row references:- - `:tables[].db_id` → `:databases[].id`- - `:fields[].table_id` → `:tables[].id`- - `:fields[].parent_id` → another `:fields[].id` in the same file- - `:fields[].fk_target_field_id` → another `:fields[].id` in the same file-- Optional columns (everything outside the always-present set per row type) are- *omitted from the row* by the export's `remove-nils` step rather than emitted- as `null`. Schemas mark these as `{:optional true} [:maybe T]` — absent and- `null` are accepted equivalently.-- `:effective_type` is special-cased: emitted only when `≠ :base_type`. Importer- treats absent as `≡ :base_type`."- (:require- [metabase.util.malli.registry :as mr]))--(set! *warn-on-reflection* true)--(mr/def ::database-info- "Database row. All three keys are required."- [:map- [:id :int]- [:name :string]- [:engine :string]])--(mr/def ::table-info- "Table row. `:id`, `:db_id`, and `:name` always present. `:schema` and- `:description` are optional and may be nil."- [:map- [:id :int]- [:db_id :int]- [:name :string]- [:schema {:optional true} [:maybe :string]]- [:description {:optional true} [:maybe :string]]])--(mr/def ::field-info- "Field row. Five required keys + several optional, per the export's- `metadata-query-format :model/Field`."- [:map- [:id :int]- [:table_id :int]- [:name :string]- [:base_type :string]- [:database_type :string]- [:parent_id {:optional true} [:maybe :int]]- [:fk_target_field_id {:optional true} [:maybe :int]]- [:description {:optional true} [:maybe :string]]- [:effective_type {:optional true} [:maybe :string]]- [:semantic_type {:optional true} [:maybe :string]]- [:coercion_strategy {:optional true} [:maybe :string]]- [:nfc_path {:optional true}- [:maybe- [:fn {:error/message "must be a sequence of strings"}- (fn [x] (and (instance? java.util.List x)- (every? string? x)))]]]])diff --git a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj b/enterprise/backend/src/metabase_enterprise/serialization/schema.cljdeleted file mode 100644index 815adb38d2a..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/schema.clj+++ /dev/null@@ -1,82 +0,0 @@-(ns metabase-enterprise.serialization.schema- "Malli schemas for the rows produced by the metadata export pipeline. References- to other entities are emitted as raw numeric ids; consumers are expected to- resolve them against the same Metabase instance."- (:require- [metabase.lib.schema.id :as lib.schema.id]- [metabase.util.malli.registry :as mr]))--(mr/def ::user-info- [:map- [:user-id pos-int?]- [:is-superuser? :boolean]])--(mr/def ::export-metadata-options- "Options accepted by the metadata export pipeline:-- - `:user-info` — caller identity for permission filtering.- - `:with-databases` — include the `databases` section.- - `:with-tables` — include the `tables` section.- - `:with-fields` — include the `fields` section.- - `:database-ids` — restrict databases to these ids.- - `:schema-ids` — `{db-id [\"schema\" ...]}` — restrict tables to these (db, schema) pairs.- - `:table-ids` — restrict tables to these ids.- - `:field-ids` — restrict fields to these ids."- [:map- [:user-info ::user-info]- [:with-databases {:optional true} [:maybe :boolean]]- [:with-tables {:optional true} [:maybe :boolean]]- [:with-fields {:optional true} [:maybe :boolean]]- [:database-ids {:optional true} [:maybe [:sequential ::lib.schema.id/database]]]- [:schema-ids {:optional true} [:maybe [:map-of ::lib.schema.id/database- [:sequential :string]]]]- [:table-ids {:optional true} [:maybe [:sequential ::lib.schema.id/table]]]- [:field-ids {:optional true} [:maybe [:sequential ::lib.schema.id/field]]]])--(mr/def ::exported-database- [:map- [:id ::lib.schema.id/database]- [:name :string]- [:engine :string]])--(mr/def ::exported-table- [:map- [:id ::lib.schema.id/table]- [:db_id ::lib.schema.id/database]- [:name :string]- [:schema {:optional true} :string]- [:description {:optional true} :string]])--(mr/def ::exported-field- [:map- [:id ::lib.schema.id/field]- [:table_id ::lib.schema.id/table]- [:name :string]- [:base_type :string]- [:description {:optional true} :string]- [:database_type {:optional true} :string]- [:effective_type {:optional true} :string]- [:semantic_type {:optional true} :string]- [:coercion_strategy {:optional true} :string]- [:nfc_path {:optional true} [:sequential :string]]- [:parent_id {:optional true} ::lib.schema.id/field]- [:fk_target_field_id {:optional true} ::lib.schema.id/field]])--(mr/def ::export-metadata-response- [:map- [:databases {:optional true} [:sequential ::exported-database]]- [:tables {:optional true} [:sequential ::exported-table]]- [:fields {:optional true} [:sequential ::exported-field]]])--(mr/def ::import-status-response- "Wire shape of `GET /metadata/import/:id`. Timestamps are ISO-8601 strings.- `:started-at`/`:finished-at`/`:wall-ms` are nil until the relevant lifecycle- stage is reached; `:error` is nil unless the import failed."- [:map {:closed true}- [:id :string]- [:status [:enum "queued" "running" "ok" "error"]]- [:enqueued-at :string]- [:started-at [:maybe :string]]- [:finished-at [:maybe :string]]- [:wall-ms [:maybe number?]]- [:error [:maybe :string]]])diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api/workspace_instance.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api/workspace_instance.cljindex 59ba690bb92..75b05213af0 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/api/workspace_instance.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api/workspace_instance.clj@@ -3,12 +3,13 @@`/api/ee/workspace-instance`. See [[metabase-enterprise.workspaces.api.workspace-manager]]for the manager-side admin operations."(:require- [metabase-enterprise.workspaces.instance :as ws.instance]+ [metabase-enterprise.workspaces.core :as ws][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.lib.schema.id :as lib.schema.id]+ [metabase.util.i18n :refer [tru]][metabase.util.malli.schema :as ms]- [toucan2.core :as t2]))+ [metabase.workspaces.core :as ws.oss]));;; ----------------------------------------------- Schemas ----------------------------------------------------@@ -25,7 +26,8 @@(def ^:private WorkspaceInstance[:map[:name ms/NonBlankString]- [:databases [:map-of ::lib.schema.id/database WorkspaceInstanceDatabase]]])+ [:databases [:map-of ::lib.schema.id/database WorkspaceInstanceDatabase]]+ [:can_write :boolean]])(def ^:private TableRemapping[:map@@ -52,7 +54,20 @@(defn- present-workspace-instance [workspace]{:name (:name workspace)- :databases (update-vals (:databases workspace) present-workspace-instance-database)})+ :databases (update-vals (:databases workspace) present-workspace-instance-database)+ :can_write (not (ws/workspace-locked-by-config?))})++;;; ------------------------------------------------ Guards ----------------------------------------------------++(defn- check-workspace-writable!+ "Throw a 400 if this instance's workspace is locked by deployment config (a+ `config.yml` `:workspace` section or the `MB_INSTANCE_WORKSPACE` env var).+ Call after `check-superuser`, so a non-admin still gets a 403 rather than this."+ []+ (when (ws/workspace-locked-by-config?)+ (throw (ex-info (tru "This instance''s workspace is set by deployment config (`config.yml` or the `MB_INSTANCE_WORKSPACE` env var) and can''t be changed at runtime. Remove that config and restart this instance to enable workspace edits.")+ {:status-code 400+ :workspace-locked-by-config true}))));;; ---------------------------------------------- Endpoints ---------------------------------------------------@@ -61,15 +76,51 @@`{:data ...}` envelope.Reads from the `instance-workspace` setting populated at boot by the `:workspace`- section of `config.yml`. `:data` is `null` when no workspace was loaded — i.e.- this is a manager-only instance, or no `config.yml` was present at boot. The- envelope avoids an empty JSON body for the `nil` case."+ section of `config.yml`, or at runtime by `POST /current`. `:data` is `null`+ when no workspace was loaded — i.e. this is a manager-only instance, or no+ `config.yml` was present at boot and `POST /current` hasn't been called.+ The envelope avoids an empty JSON body for the `nil` case.++ The workspace's `can_write` flag is `false` when it was set by deployment+ config (`config.yml` or `MB_INSTANCE_WORKSPACE`); `POST`/`DELETE` are rejected+ in that case."+ []+ (api/check-superuser)+ {:data (some-> (ws/instance-workspace) present-workspace-instance)})++(api.macros/defendpoint :post "/current" :- WorkspaceInstance+ "Install a workspace config on this instance at runtime. Accepts the same shape+ `GET /current` returns and persists it via the `instance-workspace` setting so+ it survives restarts. Use this on a running instance to enter workspace mode+ without restarting from `config.yml`.++ Returns 400 if the workspace was set by deployment config (`config.yml` or+ `MB_INSTANCE_WORKSPACE`) and so is read-only."+ [_route-params+ _query-params+ body :- ::ws.oss/workspace-instance-config]+ (api/check-superuser)+ (check-workspace-writable!)+ (ws/set-instance-workspace! body)+ (present-workspace-instance (ws/instance-workspace)))++(api.macros/defendpoint :delete "/current" :- :nil+ "Clear the workspace config on this instance. After this returns, the instance+ is no longer in workspace mode and `GET /current` returns `nil`. Also drops+ every `TableRemapping` row, since stale mappings from the prior workspace+ would otherwise keep rewriting queries on the now-unmanaged databases.++ Returns 400 if the workspace was set by deployment config (`config.yml` or+ `MB_INSTANCE_WORKSPACE`) and so is read-only."[](api/check-superuser)- {:data (some-> (ws.instance/instance-workspace) present-workspace-instance)})+ (check-workspace-writable!)+ (ws/clear-instance-workspace!)+ (ws/clear-all-remappings!)+ nil)(api.macros/defendpoint :get "/table-remappings" :- [:sequential TableRemapping]"Return all table remappings, ordered by id."[](api/check-superuser)- (mapv present-remapping (t2/select :model/TableRemapping {:order-by [[:id :asc]]})))+ (mapv present-remapping (ws/list-remappings)))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api/workspace_manager.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api/workspace_manager.cljindex 350d4349aa3..839be07a962 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/api/workspace_manager.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api/workspace_manager.clj@@ -1,35 +1,29 @@(ns metabase-enterprise.workspaces.api.workspace-manager"EE API endpoints for managing workspaces, served under `/api/ee/workspace-manager`.- Validation and presentation only — domain logic lives in the workspace models- and [[metabase-enterprise.workspaces.provisioning]]; permission predicates live- on `:model/Workspace` and `:model/WorkspaceDatabase` (see- `mi/can-read?`/`can-write?`/`can-create?`).-- Creating a workspace only inserts `:unprovisioned` rows; provisioning is- started explicitly via `POST /:id/provision` and runs in the background —- clients poll `GET /:id` and follow the workspace's `:status`."+ Validation and presentation only — domain logic lives in+ [[metabase-enterprise.workspaces.core]] and permission predicates live on+ `:model/Workspace` and `:model/WorkspaceDatabase` (see `mi/can-read?`/`can-write?`/`can-create?`)."(:require[medley.core :as m]- [metabase-enterprise.workspaces.models.workspace :as workspace]- [metabase-enterprise.workspaces.provisioning :as ws.provisioning]- [metabase-enterprise.workspaces.schema :as ws.schema]+ [metabase-enterprise.workspaces.config :as ws.config]+ [metabase-enterprise.workspaces.core :as ws][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi]- [metabase.util.i18n :refer [tru]]- [metabase.util.jvm :as u.jvm]- [metabase.util.log :as log][metabase.util.malli.schema :as ms][toucan2.core :as t2]));;; ----------------------------------------------- Schemas ----------------------------------------------------+(def ^:private WorkspaceStatus+ [:enum {:decode/api keyword}+ :unprovisioned :provisioning :provisioned :deprovisioning])+(def ^:private CreateWorkspaceParams[:map {:closed true}- [:name ms/NonBlankString]- [:target_branch {:optional true} [:maybe ms/NonBlankString]]- [:database_ids [:sequential {:min 1} ::lib.schema.id/database]]])+ [:name ms/NonBlankString]+ [:database_ids [:sequential {:min 1} ::lib.schema.id/database]]])(def ^:private UpdateWorkspaceParams[:map {:closed true}@@ -40,8 +34,7 @@[:database_id ::lib.schema.id/database][:input_schemas [:sequential ms/NonBlankString]][:output_namespace :string]- [:status ::ws.schema/workspace-database-status]- [:status_details [:maybe :string]]+ [:status WorkspaceStatus][:database {:optional true} [:maybe :map]]])(def ^:private CreatorResponse@@ -61,25 +54,20 @@(def ^:private WorkspaceResponse[:map {:closed true}- [:id ms/PositiveInt]- [:name ms/NonBlankString]- [:target_branch [:maybe :string]]- [:status ::ws.schema/workspace-status]- [:status_details [:maybe :string]]- [:instance_id [:maybe :string]]- [:instance_url [:maybe :string]]- [:creator [:maybe CreatorResponse]]- [:created_at DateTimeWithTimeZone]- [:updated_at DateTimeWithTimeZone]+ [:id ms/PositiveInt]+ [:name ms/NonBlankString]+ [:creator [:maybe CreatorResponse]]+ [:created_at DateTimeWithTimeZone]+ [:updated_at DateTimeWithTimeZone];; Both the list and GET /:id endpoints hydrate `:databases`; it stays;; optional so clients treat a missing array as `[]`.- [:databases {:optional true} [:sequential WorkspaceDatabaseResponse]]])+ [:databases {:optional true} [:sequential WorkspaceDatabaseResponse]]]);;; -------------------------------------------- Presentation --------------------------------------------------(defn- present-workspace-database [wsd](-> wsd- (select-keys [:database_id :input_schemas :output_namespace :status :status_details :database])+ (select-keys [:database_id :input_schemas :output_namespace :status :database]);; never expose connection credentials(m/update-existing :database #(some-> % (dissoc :details)))))@@ -89,7 +77,7 @@(defn- present-workspace [workspace](some-> workspace- (select-keys [:id :name :target_branch :status :status_details :instance_id :instance_url :creator :created_at :updated_at :databases])+ (select-keys [:id :name :creator :created_at :updated_at :databases])(update :creator present-creator)(m/update-existing :databases #(mapv present-workspace-database %))))@@ -101,92 +89,57 @@(api/check-superuser)(into [] (comp (filter mi/can-read?)(map present-workspace))- (-> (t2/select :model/Workspace {:order-by [[:id :asc]]})- (t2/hydrate :creator [:databases :database]))))+ (ws/list-workspaces)))(api.macros/defendpoint :get "/:id" :- WorkspaceResponse"Get a single Workspace by id."[{:keys [id]} :- [:map [:id ms/PositiveInt]]]- (let [ws (api/read-check :model/Workspace id)]- (-> ws- (t2/hydrate :creator [:databases :database])- present-workspace)))+ (api/read-check :model/Workspace id)+ (present-workspace (api/check-404 (ws/get-workspace id))))(api.macros/defendpoint :post "/" :- WorkspaceResponse"Create a new Workspace attached to the given databases (each must be eligible- for workspaces). The workspace and its databases start `:unprovisioned` —- start provisioning explicitly via `POST /:id/provision`."- [_route-params _query-params {:keys [target_branch database_ids] :as params} :- CreateWorkspaceParams]+ for workspaces) and provision it (blocking)."+ [_route-params _query-params params :- CreateWorkspaceParams](api/create-check :model/Workspace params)- (when target_branch- (api/check-400 (not (t2/exists? :model/Workspace :target_branch target_branch))- (tru "A workspace with this target branch already exists")))- (-> (workspace/create-workspace! {:name (:name params)- :target_branch target_branch- :creator_id api/*current-user-id*- :databases (workspace/workspace-databases database_ids)})- (t2/hydrate :creator [:databases :database])- present-workspace))+ (present-workspace+ (ws/create-workspace!+ (assoc params :creator_id api/*current-user-id*))))(api.macros/defendpoint :put "/:id" :- WorkspaceResponse"Update a workspace's name."[{:keys [id]} :- [:map [:id ms/PositiveInt]]_query-paramsparams :- UpdateWorkspaceParams]- (let [ws (api/write-check :model/Workspace id)- ws (if-let [new-name (:name params)]- (do (t2/update! :model/Workspace :id id {:name new-name})- (assoc ws :name new-name))- ws)]- (-> ws- (t2/hydrate :creator [:databases :database])- present-workspace)))+ (api/write-check :model/Workspace id)+ (when (:name params)+ (t2/update! :model/Workspace :id id {:name (:name params)}))+ (present-workspace (api/check-404 (ws/get-workspace id))))(api.macros/defendpoint :delete "/:id" :- :nil- "Delete a Workspace. Refuses with a 400 unless the workspace is strictly- `:unprovisioned` — deprovision it first (`POST /:id/deprovision`). The status- condition sits on the DELETE itself, closing the race with a concurrently- started run. The workspace_database rows are cascade-deleted with the- workspace. 204 on success, no response body."+ "Delete a Workspace. Tears down every database's warehouse isolation first+ (blocking, any state). Each database is either fully torn down (its row is+ deleted) or kept; when any teardown fails the workspace is kept too — with the+ failed rows, so the delete can be retried — and the combined failure is+ returned as the error. 204 on success, no response body."[{:keys [id]} :- [:map [:id ms/PositiveInt]]](api/write-check :model/Workspace id)- (api/check-400 (pos? (t2/delete! :model/Workspace :id id :status :unprovisioned)))+ (ws/delete-workspace! id)nil)-(api.macros/defendpoint :post "/:id/provision" :- WorkspaceResponse- "Start provisioning the workspace in the background and return immediately.- Retryable from any settled status except `:provisioned`; 400 when already- provisioned or while a provision or deprovision run is in flight. Poll- `GET /:id` to follow the workspace's `:status`; the response reflects the- just-started run (`:database-provisioning`)."- [{:keys [id]} :- [:map [:id ms/PositiveInt]]]- (let [ws (api/write-check :model/Workspace id)- ws (ws.provisioning/set-initial-workspace-provisioning-status! ws)- response (-> ws- (t2/hydrate :creator [:databases :database])- present-workspace)]- (u.jvm/in-virtual-thread*- (try- (ws.provisioning/provision-workspace! ws)- (catch Throwable t- (log/error t "Async workspace provisioning failed"))))- response))--(api.macros/defendpoint :post "/:id/deprovision" :- WorkspaceResponse- "Start deprovisioning the workspace in the background and return immediately.- Retryable from any settled status except `:unprovisioned`; 400 when already- unprovisioned or while a provision or deprovision run is in flight. Poll- `GET /:id` to follow the workspace's `:status`; the response reflects the- just-started run (`:instance-deprovisioning`)."+;;; ------------------------------------------- Config download --------------------------------------------------++(api.macros/defendpoint :get "/:id/config"+ :- [:map+ [:status [:= 200]]+ [:headers [:map-of :string :string]]+ [:body :string]]+ "Download the workspace's developer-instance config as a YAML file. 409 if any+ of the workspace's databases is not `:provisioned`."[{:keys [id]} :- [:map [:id ms/PositiveInt]]]- (let [ws (api/write-check :model/Workspace id)- ws (ws.provisioning/set-initial-workspace-deprovisioning-status! ws)- response (-> ws- (t2/hydrate :creator [:databases :database])- present-workspace)]- (u.jvm/in-virtual-thread*- (try- (ws.provisioning/deprovision-workspace! ws)- (catch Throwable t- (log/error t "Async workspace deprovisioning failed"))))- response))+ (api/write-check :model/Workspace id)+ (let [config (api/check-404 (ws.config/build-workspace-config id))]+ {:status 200+ :headers {"Content-Type" "application/x-yaml"+ "Content-Disposition" "attachment; filename=\"config.yml\""}+ :body (ws.config/config->yaml config)}))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/config.clj b/enterprise/backend/src/metabase_enterprise/workspaces/config.cljindex 9ce6836ed71..78cdffc6bf5 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/config.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/config.clj@@ -1,10 +1,9 @@(ns metabase-enterprise.workspaces.config(:require[clojure.string :as str]- [metabase-enterprise.remote-sync.core :as remote-sync]+ [metabase-enterprise.workspaces.core :as ws]+ [metabase-enterprise.workspaces.models.workspace :as workspace][metabase-enterprise.workspaces.models.workspace-database]- [metabase-enterprise.workspaces.provisioning.database :as provisioning.database]- [metabase-enterprise.workspaces.settings :as ws.settings][metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.sample-data.core :as sample-data]@@ -74,7 +73,7 @@mapping."[db output-namespace](let [components (set (driver/qualified-name-components (:engine db)))- positions (provisioning.database/engine-namespace-positions db)+ positions (ws/engine-namespace-positions db)schema (when-not (str/blank? output-namespace) output-namespace)](cond-> {}(:db components) (assoc :db (:db positions))@@ -106,36 +105,6 @@[:not-in :id workspace-db-ids]true)]}))-(defn- remote-sync-settings- "Remote-sync settings for the child instance's `settings:` config section, or- nil when remote sync is not enabled on this instance. The child instance- always gets `:read-write` — its whole purpose is committing changes back.- The child works on the workspace's `:target_branch` when set, falling back- to this instance's configured remote-sync branch."- [workspace]- (when (remote-sync/remote-sync-enabled)- (let [target-branch (:target_branch workspace)]- {:remote-sync-url (remote-sync/remote-sync-url)- :remote-sync-branch (if (str/blank? target-branch)- (remote-sync/remote-sync-branch)- target-branch)- :remote-sync-token (remote-sync/remote-sync-token)- :remote-sync-type :read-write})))--(defn- user-entries- "config-file `users:` entries for the child instance: the workspace creator- as a superuser, when both the creator and the configured- `workspace-instance-user-password` exist."- [workspace]- (let [{:keys [creator]} (t2/hydrate workspace :creator)- password (ws.settings/workspace-instance-user-password)]- (when (and creator (not (str/blank? password)))- [{:first_name (:first_name creator)- :last_name (:last_name creator)- :email (:email creator)- :password password- :is_superuser true}])))-(defn- sample-database-entries"If the instance has a Sample Database, emit a single placeholder entry theimport side will use to recreate it. Always uses the canonical name/engine —@@ -148,7 +117,7 @@:is_sample true}]))(defn build-workspace-config- "Return a downloadable config.yml-shaped map for `workspace`:+ "Return a downloadable config.yml-shaped map for `workspace-id`:{:version 1:config {:databases [...]@@ -160,41 +129,33 @@WorkspaceDatabase's override credentials and adds `schema-filters-*` keysderived from `:input_schemas`. Per-database workspace entries carry theexpanded `{:db ?, :schema ?}` namespace map directly — the same shape the- `instance-workspace` setting stores. When remote sync is enabled on this- instance, `:config` also carries a `:settings` section pointing the child- instance at the same git repo in `:read-write` mode. When the- `workspace-instance-user-password` setting is set and the workspace has a- creator, `:config` carries a `:users` section creating that user as a- superuser on the child. Takes a workspace with- its `:databases` hydrated. Throws a 409 `ex-info` if any of the workspace's- databases is not `:provisioned`."- [workspace]- (let [wsds (:databases workspace)]- (when (some #(not= :provisioned (:status %)) wsds)- (throw (ex-info "Cannot build config while workspace has databases that are not :provisioned"- {:status-code 409- :workspace_id (:id workspace)})))- (let [workspace-db-ids (mapv :database_id wsds)- dbs-by-id (if-let [ids (seq workspace-db-ids)]- (into {} (map (juxt :id identity))- (t2/select :model/Database :id [:in ids]))- {})- pairs (for [wsd wsds- :let [db (get dbs-by-id (:database_id wsd))]]- [wsd db])- ws-entries (mapv (fn [[wsd db]] (database-entry wsd db)) pairs)- stub-entries (mapv stub-database-entry (stub-databases workspace-db-ids))- sample-entries (sample-database-entries)- settings (remote-sync-settings workspace)- users (user-entries workspace)]- {:version 1- :config (cond-> {:databases (-> ws-entries- (into stub-entries)- (into sample-entries))- :workspace {:name (:name workspace)- :databases (into {} (map (fn [[wsd db]] (workspace-database-entry wsd db))) pairs)}}- settings (assoc :settings settings)- users (assoc :users users))})))+ `instance-workspace` setting stores. Returns nil when the workspace does not+ exist. Throws a 409 `ex-info` if any of the workspace's databases is not+ `:provisioned`."+ [workspace-id]+ (when-let [ws (workspace/get-workspace workspace-id)]+ (let [wsds (:databases ws)]+ (when (some #(not= :provisioned (:status %)) wsds)+ (throw (ex-info "Cannot build config while workspace has databases that are not :provisioned"+ {:status-code 409+ :workspace_id workspace-id})))+ (let [workspace-db-ids (mapv :database_id wsds)+ dbs-by-id (if-let [ids (seq workspace-db-ids)]+ (into {} (map (juxt :id identity))+ (t2/select :model/Database :id [:in ids]))+ {})+ pairs (for [wsd wsds+ :let [db (get dbs-by-id (:database_id wsd))]]+ [wsd db])+ ws-entries (mapv (fn [[wsd db]] (database-entry wsd db)) pairs)+ stub-entries (mapv stub-database-entry (stub-databases workspace-db-ids))+ sample-entries (sample-database-entries)]+ {:version 1+ :config {:databases (-> ws-entries+ (into stub-entries)+ (into sample-entries))+ :workspace {:name (:name ws)+ :databases (into {} (map (fn [[wsd db]] (workspace-database-entry wsd db))) pairs)}}}))))(defn config->yaml"Render a workspace config map as a pretty-printed YAML string."diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/core.clj b/enterprise/backend/src/metabase_enterprise/workspaces/core.cljindex 7935ccbef17..faecf90ce24 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/core.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/core.clj@@ -1,13 +1,309 @@(ns metabase-enterprise.workspaces.core- "Facade for the workspaces module: the only namespace code outside the module- may require. Everything here is a re-export; the implementations live in the- module-internal namespaces (`instance`, `provisioning`, `models.*`, ...)."+ "Programmatic API for workspaces.++ Two operational surfaces share this namespace:++ - **Manager-side state.** Workspaces and their per-database configs live in the+ `:model/Workspace` and `:model/WorkspaceDatabase` rows. Mutations go through+ [[create-workspace!]] and [[delete-workspace!]]. Provisioning operations are+ synchronous — the caller waits until the warehouse work completes.++ - **Instance-side state.** When a Metabase boots in workspace mode, the+ `:workspace` section loader (`metabase-enterprise.advanced-config.file.workspace`)+ parses `config.yml` and stores the resulting workspace map in the+ `instance-workspace` setting. Workspace-aware code (transform target+ rewriting, table-remapping QP middleware) reads from the setting via+ [[workspace-mode?]] / [[db-workspace-namespace]]. The setting lives in the+ instance's own application database — child instances have their own app DB,+ so storing it there persists the workspace across restarts without leaking+ between parent and child.++ Per-database lifecycle for the manager-side rows:++ provision+ unprovisioned ────────────► provisioning ──────► provisioned+ ▲ │ │+ │ failure │ │ deprovision+ │ ▼ ▼+ │ unprovisioned ◄──── deprovisioning+ │ │+ │ failure │+ │ ▼+ └──────────────────────────────────────── provisioned"(:require- [metabase-enterprise.workspaces.instance]- [metabase.util.namespaces :as shared.ns]))--(shared.ns/import-fns- [metabase-enterprise.workspaces.instance- clear-instance-workspace!- instance-workspace- set-instance-workspace!])+ [clojure.string :as str]+ [metabase-enterprise.workspaces.models.workspace :as workspace]+ [metabase-enterprise.workspaces.models.workspace-database :as workspace-database]+ [metabase-enterprise.workspaces.provisioning :as provisioning]+ [metabase-enterprise.workspaces.settings :as ws.settings]+ [metabase.driver.sql :as driver.sql]+ [metabase.premium-features.core :refer [defenterprise]]+ [metabase.settings.core :as setting]+ [metabase.util.malli :as mu]+ [metabase.workspaces.core :as ws]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(defn engine-namespace-positions+ "Return `{:db ?, :schema ?}` — the values that should populate the `:db` and+ `:schema` AST slots for a Table row in `database`. `table` is optional; pass+ it when you want the schema position derived from the Table's `:schema`+ column (the normal `spec-for-table` case). Pass nil for `table` when you only+ need the `:db` slot (workspace `output_namespace` expansion, GRANT emission).++ `nil` for either slot means \"this driver doesn't emit this AST level.\"+ Empty-string sentinel coercion happens at the storage boundary, not here.++ Driven by [[metabase.driver.sql/table-qualification-style]] ++ [[metabase.driver.sql/db-slot-value]] -- third-party drivers participate by+ implementing those rather than getting a new case branch here."+ ([database] (engine-namespace-positions database nil))+ ([database table]+ (case (driver.sql/table-qualification-style (:engine database))+ :table-qualification-style/table+ {:db nil+ :schema nil}++ :table-qualification-style/schema-table+ {:db nil+ :schema (:schema table)}++ :table-qualification-style/db-table+ {:db (driver.sql/db-slot-value (:engine database) database)+ :schema nil}++ :table-qualification-style/db-schema-table+ {:db (driver.sql/db-slot-value (:engine database) database)+ :schema (:schema table)})))++(defn- coerce-database-id-key+ "JSON round-trips through the `instance-workspace` setting return integer+ `Database.id` keys as keywords (e.g. `:1`). Coerce them back to ints."+ [k]+ (cond+ (int? k) k+ (keyword? k) (parse-long (name k))+ (string? k) (parse-long k)))++(defn- normalize-database-keys+ "Coerce the `:databases` map keys to ints. See [[coerce-database-id-key]]."+ [config]+ (some-> config+ (update :databases #(update-keys % coerce-database-id-key))))++(mu/defn set-instance-workspace! :- :any+ "Store the workspace config in the `instance-workspace` setting. Replaces any+ prior value. The shape is validated against `::ws/workspace-instance-config`."+ [config :- ::ws/workspace-instance-config]+ (ws.settings/instance-workspace! config)+ nil)++(defn clear-instance-workspace!+ "Clear the `instance-workspace` setting."+ []+ (ws.settings/instance-workspace! nil)+ nil)++;;; ------------------------------- Instance-workspace lock -------------------------------++(defonce ^:private locked-by-config?* (atom false))++(defn workspace-locked-by-config?+ "True iff this instance's workspace was set by out-of-band deployment config —+ a `:workspace` section in `config.yml` at boot, or the `MB_INSTANCE_WORKSPACE`+ env var — and so must not be mutated through the workspace-instance API. The+ lock is per-boot and not persisted: removing the config and restarting clears it.++ Read at the HTTP boundary only. Do NOT consult it from `set-instance-workspace!`+ or `clear-instance-workspace!` — internal callers and test fixtures rely on+ those setters staying unconditional."+ []+ (or @locked-by-config?*+ ;; Checked inline rather than captured at boot like the atom: nothing reads+ ;; this env var at startup, and the lookup is cheap.+ (some? (setting/env-var-value :instance-workspace))))++(defn mark-locked-by-config!+ "Called by the boot loader only; runtime code paths never flip the lock."+ []+ (reset! locked-by-config?* true))++(defn clear-all-remappings!+ "Delete every `:model/TableRemapping` row across all databases."+ []+ (t2/delete! :model/TableRemapping)+ nil)++(defn instance-workspace+ "Return the workspace loaded on this instance, or nil if none."+ []+ (normalize-database-keys (ws.settings/instance-workspace)))++(defenterprise workspace-mode?+ "EE impl: true iff this instance is running in workspace mode (the+ `instance-workspace` setting is populated — either from a `:workspace` section+ of `config.yml` at boot or by a `POST /api/ee/advanced-config`). Single+ source of truth for gating features that conflict with workspace remapping+ (DB routing, impersonation, writeback, CSV upload, model persistence). Use+ [[db-workspace-namespace]] when you need per-database scoping.++ Deliberately ungated on premium features: a workspace child instance bootstraps+ from `config.yml` *before* its token is installed; if the workspace map is+ loaded, we refuse incompatible features regardless of token state."+ :feature :none+ []+ (some? (ws.settings/instance-workspace)))++(defn db-workspace-namespace+ "Return the workspace-isolated output namespace map for `db-id` on this+ instance, or `nil` when this instance is not running a workspace or the+ workspace has no entry for `db-id`. The namespace map is+ `{:db ?, :schema ?}` - either or both keys may be absent depending on+ the driver's `qualified-name-components`. Reads from the `workspace-instance`+ setting."+ [db-id]+ (get-in (instance-workspace) [:databases db-id :output]))++(defn list-remappings+ "Return all TableRemapping rows, ordered by id."+ []+ (t2/select :model/TableRemapping {:order-by [[:id :asc]]}))++;;; ------------------------------------- Manager-side helpers ------------------------------------------------++(defn- assert-workspace-exists [workspace-id]+ (when-not (t2/exists? :model/Workspace :id workspace-id)+ (throw (ex-info "Workspace not found"+ {:status-code 404 :workspace_id workspace-id}))))++(defn- assert-database-exists [database-id]+ (or (t2/select-one :model/Database :id database-id)+ (throw (ex-info "Database not found"+ {:status-code 404 :database_id database-id}))))++(defn- assert-database-eligible-for-workspaces [database]+ (when-not (workspace-database/database-eligible-for-workspaces? database)+ (throw (ex-info "Workspaces are not enabled for this database"+ {:status-code 400 :database_id (:id database)}))))++;;; ------------------------------------- Manager-side reads --------------------------------------------------++(defn get-workspace+ "Return the Workspace with the given `id`, hydrated with `:databases` and `:creator`.+ Returns nil if not found."+ [id]+ (workspace/get-workspace id))++(defn list-workspaces+ "Return all Workspaces, each hydrated with `:databases` and `:creator`."+ []+ (workspace/list-workspaces))++;;; ------------------------------------- Manager-side writes -------------------------------------------------++(defn- combined-exception+ "One exception carrying every failure in `throwables`: the first is the cause,+ the rest are attached as suppressed (the JDK mechanism for multiple failures+ on one throwable). The message is the messages the databases returned, joined+ into one line. The ex-data is shaped so the API error middleware returns just+ that message — without it, an uncaught 500 dumps the full `Throwable->map`+ cause chain into the response."+ ^Throwable [throwables]+ (let [message (str/join "; " (map #(or (ex-message %) (str (class %))) throwables))+ ^Throwable ex (ex-info message+ {:status-code 500+ :message message+ :errors {:_error message}}+ (first throwables))]+ (run! #(.addSuppressed ex ^Throwable %) (rest throwables))+ ex))++(defn- provision-workspace-databases!+ "Provision every WorkspaceDatabase of the hydrated workspace `ws` synchronously+ (blocking), failing fast on the first error."+ [ws]+ (doseq [{wsd-id :id} (:databases ws)]+ (provisioning/provision-single! wsd-id)))++(defn- teardown-workspace-databases!+ "Tear down every WorkspaceDatabase of `workspace-id` — any state, blocking.+ Mirrors [[provision-workspace-databases!]], but continues past failures so each+ row gets its attempt: rows whose teardown succeeds are deleted immediately+ (progress is persisted per row, so an instance crash midway loses nothing);+ rows whose teardown fails are kept. Throws the combined failures when any+ teardown failed."+ [workspace-id]+ (let [failures (into []+ (keep (fn [wsd]+ (try+ (provisioning/teardown-workspace-database! wsd provisioning/dispatching-provisioner)+ nil+ (catch Throwable t t))))+ (t2/select :model/WorkspaceDatabase :workspace_id workspace-id))]+ (when (seq failures)+ (throw (combined-exception failures)))))++(defn create-workspace!+ "Create a new Workspace, attach the databases with ids `database_ids` — each must+ exist (404) and be eligible for workspaces (400, see+ [[workspace-database/database-eligible-for-workspaces?]]) — with all of their+ known schemas as `input_schemas`, and provision each database (blocking).+ Returns the created workspace, hydrated.++ The workspace and its WorkspaceDatabase rows are committed BEFORE provisioning+ starts — deliberately NOT one big transaction. The rows are the durable record+ of warehouse resources that may (partially) exist, so they must survive an+ instance crash mid-provision; a rollback would erase them while the warehouse+ objects live on. Cleanup after a provisioning failure is therefore explicit:+ every database is torn down; when the cleanup removes everything, the Workspace+ row is deleted and the provisioning error is rethrown; when the cleanup itself+ fails, the workspace is kept — and returned like a successful create — with the+ failed rows (`:unprovisioned`) so the leak stays visible and the teardown can+ be retried via delete. Callers detect that case from the databases' statuses."+ [{:keys [name creator_id database_ids]}]+ (let [databases (mapv (fn [db-id]+ (let [database (assert-database-exists db-id)]+ (assert-database-eligible-for-workspaces database)+ {:database_id db-id+ :input_schemas (workspace-database/database-input-schemas database)}))+ database_ids)+ ws (workspace/create-workspace! {:name name+ :creator_id creator_id+ :databases databases})]+ (try+ (provision-workspace-databases! ws)+ (catch Throwable t+ (let [cleaned-up? (try+ (teardown-workspace-databases! (:id ws))+ true+ ;; Cleanup failed: the workspace and the failed rows are+ ;; kept — and returned — so the teardown can be retried+ ;; via delete. Both errors are deliberately swallowed,+ ;; not logged (raw warehouse errors are sensitive);+ ;; callers see the failure in the databases' statuses,+ ;; and a delete retry surfaces the teardown errors.+ (catch Throwable _ false))]+ (when cleaned-up?+ (try+ (workspace/delete-workspace! (:id ws))+ (catch Throwable delete-t+ (.addSuppressed t delete-t)))+ (throw (combined-exception [t]))))))+ (workspace/get-workspace (:id ws))))++(defn delete-workspace!+ "Tear down every database's warehouse isolation (any state, blocking), then+ delete the workspace. There is no partial deletion: each WorkspaceDatabase is+ either fully torn down (warehouse footprint confirmed gone, row deleted) or+ kept. Every database gets its teardown attempt even when earlier ones fail; if+ any of them fail, the workspace is kept alongside the failed rows and one+ exception combining all the failures is thrown, so the delete can be retried.+ Progress is persisted per row, so an instance crash midway loses nothing.+ App-DB `TableRemapping` rows are always cleared, so query routing is never+ left dangling. Returns nil."+ [id]+ (assert-workspace-exists id)+ (teardown-workspace-databases! id)+ (workspace/delete-workspace! id)+ nil)diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/init.clj b/enterprise/backend/src/metabase_enterprise/workspaces/init.cljindex 1e4c887e0b9..db1fbe6533e 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/init.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/init.clj@@ -10,7 +10,6 @@app-db. See `metabase-enterprise.core.init` docstring for the broaderpattern."(:require- [metabase-enterprise.workspaces.instance][metabase-enterprise.workspaces.models.workspace][metabase-enterprise.workspaces.models.workspace-database][metabase-enterprise.workspaces.query-processor.middleware]diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/instance.clj b/enterprise/backend/src/metabase_enterprise/workspaces/instance.cljdeleted file mode 100644index 36a9acda6da..00000000000--- a/enterprise/backend/src/metabase_enterprise/workspaces/instance.clj+++ /dev/null@@ -1,75 +0,0 @@-(ns metabase-enterprise.workspaces.instance- "Instance-side workspace state. When a Metabase boots in workspace mode, the- `:workspace` section loader (`metabase-enterprise.advanced-config.file.workspace`)- parses `config.yml` and stores the resulting workspace map in the- `instance-workspace` setting. Workspace-aware code (transform target- rewriting, table-remapping QP middleware) reads from the setting via- [[workspace-mode?]] / [[db-workspace-namespace]]. The setting lives in the- instance's own application database — child instances have their own app DB,- so storing it there persists the workspace across restarts without leaking- between parent and child."- (:require- [metabase-enterprise.workspaces.settings :as ws.settings]- [metabase.premium-features.core :refer [defenterprise]]- [metabase.util.malli :as mu]- [metabase.workspaces.core :as ws]))--(set! *warn-on-reflection* true)--(defn- coerce-database-id-key- "JSON round-trips through the `instance-workspace` setting return integer- `Database.id` keys as keywords (e.g. `:1`). Coerce them back to ints."- [k]- (cond- (int? k) k- (keyword? k) (parse-long (name k))- (string? k) (parse-long k)))--(defn- normalize-database-keys- "Coerce the `:databases` map keys to ints. See [[coerce-database-id-key]]."- [config]- (some-> config- (update :databases #(update-keys % coerce-database-id-key))))--(mu/defn set-instance-workspace! :- :nil- "Store the workspace config in the `instance-workspace` setting. Replaces any- prior value. The shape is validated against `::ws/workspace-instance-config`."- [config :- ::ws/workspace-instance-config]- (ws.settings/instance-workspace! config)- nil)--(defn clear-instance-workspace!- "Clear the `instance-workspace` setting."- []- (ws.settings/instance-workspace! nil)- nil)--(defn instance-workspace- "Return the workspace loaded on this instance, or nil if none."- []- (normalize-database-keys (ws.settings/instance-workspace)))--(defenterprise workspace-mode?- "EE impl: true iff this instance is running in workspace mode (the- `instance-workspace` setting is populated — either from a `:workspace` section- of `config.yml` at boot or by a `POST /api/ee/advanced-config`). Single- source of truth for gating features that conflict with workspace remapping- (DB routing, impersonation, writeback, CSV upload, model persistence). Use- [[db-workspace-namespace]] when you need per-database scoping.-- Deliberately ungated on premium features: a workspace child instance bootstraps- from `config.yml` *before* its token is installed; if the workspace map is- loaded, we refuse incompatible features regardless of token state."- :feature :none- []- (some? (ws.settings/instance-workspace)))--(defn db-workspace-namespace- "Return the workspace-isolated output namespace map for `db-id` on this- instance, or `nil` when this instance is not running a workspace or the- workspace has no entry for `db-id`. The namespace map is- `{:db ?, :schema ?}` - either or both keys may be absent depending on- the driver's `qualified-name-components`. Reads from the `instance-workspace`- setting."- [db-id]- (get-in (instance-workspace) [:databases db-id :output]))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace.clj b/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace.cljindex de0480310b7..1e139db39ee 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace.clj@@ -1,22 +1,20 @@(ns metabase-enterprise.workspaces.models.workspace(:require- [metabase-enterprise.workspaces.models.workspace-database :as workspace-database]+ [metabase-enterprise.workspaces.models.workspace-database][metabase.api.common :as api][metabase.models.interface :as mi][metabase.util :as u][methodical.core :as methodical][toucan2.core :as t2]))+(comment metabase-enterprise.workspaces.models.workspace-database/keep-me)+(methodical/defmethod t2/table-name :model/Workspace [_model] :workspace)(doto :model/Workspace(derive :metabase/model)(derive :hook/timestamped?))-(t2/deftransforms :model/Workspace- {:status mi/transform-keyword- :status_details mi/transform-encrypted-text})-;;; --------------------------------------- Permission predicates ---------------------------------------;;;;;; Workspaces are superuser-only: read, write, and create all require admin.@@ -55,6 +53,27 @@:id [:in ids])))):creator_id))+(defn list-workspaces+ "Return every Workspace with its `:databases` (each with its `:database`) and+ `:creator` hydrated."+ []+ (t2/hydrate (t2/select :model/Workspace {:order-by [[:id :asc]]}) :creator [:databases :database]))++(defn get-workspace+ "Return the Workspace with the given id and its `:databases` (each with its+ `:database`) + `:creator` hydrated, or nil if none exists."+ [id]+ (when-let [workspace (t2/select-one :model/Workspace :id id)]+ (t2/hydrate workspace :creator [:databases :database])))++(defn get-workspace-by-name+ "Return the Workspace with the given name and its `:databases` + `:creator` hydrated,+ or nil if none exists. Workspace names are not unique at the schema level, so this+ returns the lowest-id match."+ [workspace-name]+ (when-let [workspace (t2/select-one :model/Workspace :name workspace-name {:order-by [[:id :asc]]})]+ (t2/hydrate workspace :creator :databases)))+(defn- with-workspace-database-defaults"Fill server-managed columns that are NOT NULL in the DB with their defaults when callers omit them."[wsd workspace-id]@@ -63,39 +82,69 @@wsd{:workspace_id workspace-id}))-(defn- assert-database-eligible-for-workspaces [database]- (when-not (workspace-database/database-eligible-for-workspaces? database)- (throw (ex-info "Workspaces are not enabled for this database"- {:status-code 400 :database_id (:id database)}))))--(defn workspace-databases- "Build the WorkspaceDatabase param maps for `database-ids` — each database must- exist (404) and be eligible for workspaces (400) — with all of its known- schemas as `input_schemas`. Fetches the databases and their schemas with one- query each instead of per-id lookups."- [database-ids]- (let [dbs-by-id (when (seq database-ids)- (t2/select-pk->fn identity :model/Database :id [:in database-ids]))- schemas-by-id (workspace-database/database-input-schemas-by-id (keys dbs-by-id))]- (mapv (fn [db-id]- (let [database (or (get dbs-by-id db-id)- (throw (ex-info "Database not found"- {:status-code 404 :database_id db-id})))]- (assert-database-eligible-for-workspaces database)- {:database_id db-id- :input_schemas (get schemas-by-id db-id [])}))- database-ids)))-(defn create-workspace!"Create a Workspace and its nested WorkspaceDatabase rows in a single transaction.- The param map must supply `:creator_id`. Returns the created Workspace row."- [{:keys [name target_branch creator_id databases]}]+ The param map must supply `:creator_id`. Returns the created Workspace with+ `:databases` and `:creator` hydrated."+ [{:keys [name creator_id databases]}](t2/with-transaction [_conn]- (let [workspace (t2/insert-returning-instance! :model/Workspace- {:name name- :target_branch target_branch- :creator_id creator_id})]+ (let [workspace-id (t2/insert-returning-pk! :model/Workspace+ {:name name+ :creator_id creator_id})](when (seq databases)(t2/insert! :model/WorkspaceDatabase- (map #(with-workspace-database-defaults % (:id workspace)) databases)))- workspace)))+ (map #(with-workspace-database-defaults % workspace-id) databases)))+ (get-workspace workspace-id))))++(defn- reject-active-modification!+ "Throw a 409 if any row in `existing-active` (everything other than `:unprovisioned`)+ is missing from the incoming `:databases` list or would have its `:input_schemas` changed.+ Only `:unprovisioned` rows are freely mutable; `:provisioning`, `:provisioned`,+ and `:deprovisioning` rows must be preserved verbatim."+ [existing-active incoming-by-db-id]+ (doseq [{:keys [database_id input_schemas]} existing-active]+ (let [incoming (get incoming-by-db-id database_id)]+ (when (or (nil? incoming)+ (not= (vec input_schemas) (vec (:input_schemas incoming))))+ (throw (ex-info "Cannot modify a workspace_database that is not :unprovisioned"+ {:status-code 409+ :database_id database_id}))))))++(defn delete-workspace!+ "Delete a Workspace. Refuses with a 409 if any of its WorkspaceDatabase rows is in+ a non-`:unprovisioned` state (`:provisioning`, `:provisioned`, or `:deprovisioning`)+ — those point at live warehouse resources and must be unprovisioned explicitly+ first. Cascade-deletes `:unprovisioned` children via the FK."+ [id]+ (when (t2/exists? :model/WorkspaceDatabase+ :workspace_id id+ :status [:not= :unprovisioned])+ (throw (ex-info "Cannot delete a workspace with databases that are not :unprovisioned; deprovision them first"+ {:status-code 409+ :workspace_id id})))+ (t2/delete! :model/Workspace :id id))++(defn update-workspace!+ "Update a Workspace and reconcile its `WorkspaceDatabase` rows with the provided+ list. Only `:unprovisioned` rows are freely mutable. Rows in any other state+ (`:provisioning`, `:provisioned`, `:deprovisioning`) must be preserved by+ `database_id` with matching `:input`, or a 409 is raised. Fields not+ present in `params` are left untouched."+ [id params]+ (t2/with-transaction [_conn]+ (when (contains? params :name)+ (t2/update! :model/Workspace :id id {:name (:name params)}))+ (when (contains? params :databases)+ (let [databases (:databases params)+ existing (t2/select :model/WorkspaceDatabase :workspace_id id)+ active (remove #(= :unprovisioned (:status %)) existing)+ unprovisioned (filter #(= :unprovisioned (:status %)) existing)+ incoming-by-db-id (into {} (map (juxt :database_id identity)) databases)+ preserved-db-ids (set (map :database_id active))]+ (reject-active-modification! active incoming-by-db-id)+ (when-let [to-delete-ids (seq (map :id unprovisioned))]+ (t2/delete! :model/WorkspaceDatabase :id [:in to-delete-ids]))+ (when-let [to-insert (seq (remove #(contains? preserved-db-ids (:database_id %)) databases))]+ (t2/insert! :model/WorkspaceDatabase+ (map #(with-workspace-database-defaults % id) to-insert)))))+ (get-workspace id)))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace_database.clj b/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace_database.cljindex b095713ab6e..938c5bdf768 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace_database.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace_database.clj@@ -18,8 +18,7 @@(t2/deftransforms :model/WorkspaceDatabase{:database_details mi/transform-encrypted-json:input_schemas mi/transform-json- :status mi/transform-keyword- :status_details mi/transform-encrypted-text})+ :status mi/transform-keyword});;; --------------------------------------- Permission predicates ---------------------------------------;;;@@ -59,20 +58,14 @@(boolean (setting/with-database database(ws.settings/database-enable-workspaces)))))-(defn database-input-schemas-by-id- "Map of database id -> the distinct non-blank schema names of its active- tables, sorted — one query for all `database-ids`. Schemaless drivers (e.g.- MySQL) get no entry."- [database-ids]- (when (seq database-ids)- (-> (group-by :db_id (t2/select [:model/Table :db_id :schema]- :db_id [:in database-ids]- :active true))- (update-vals (fn [tables]- (->> (into #{} (map :schema) tables)- (remove #(or (nil? %) (= % "")))- sort- vec))))))+(defn database-input-schemas+ "The distinct non-blank schema names of `database`'s active tables, sorted.+ Empty for schemaless drivers (e.g. MySQL)."+ [database]+ (->> (t2/select-fn-set :schema :model/Table :db_id (:id database) :active true)+ (remove #(or (nil? %) (= % "")))+ sort+ vec))(defenterprise reconcile-workspace-database-refs-before-delete!"Enterprise impl of the `:model/Database` before-delete hook. Refuses (409) when anydiff --git a/enterprise/backend/src/metabase_enterprise/workspaces/provisioning.clj b/enterprise/backend/src/metabase_enterprise/workspaces/provisioning.cljindex 64afa18119a..ce56fcfc9e4 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/provisioning.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/provisioning.clj@@ -1,128 +1,304 @@(ns metabase-enterprise.workspaces.provisioning- "Top-level workspace provisioning orchestration: [[provision-workspace!]] and- [[deprovision-workspace!]], recording progress in the workspace's- `:status`/`:status_details` (see- `:metabase-enterprise.workspaces.schema/workspace-status`). The API runs- them in the background."(:require- [metabase-enterprise.workspaces.provisioning.database :as provisioning.database]- [metabase-enterprise.workspaces.provisioning.instance :as provisioning.instance]- [metabase-enterprise.workspaces.provisioning.remote-sync :as provisioning.remote-sync]- [metabase-enterprise.workspaces.schema :as ws.schema]- [metabase.util.malli :as mu]+ [metabase-enterprise.workspaces.models.workspace-database]+ [metabase-enterprise.workspaces.remapping-cleanup :as ws.remapping-cleanup]+ [metabase.app-db.cluster-lock :as cluster-lock]+ [metabase.driver :as driver]+ [metabase.driver.connection :as driver.conn]+ [metabase.driver.util :as driver.u]+ [metabase.util.log :as log]+ [potemkin.types :as p]+ [toucan2.connection :as t2.connection][toucan2.core :as t2]))-(set! *warn-on-reflection* true)--(mu/defn- set-initial-workspace-status! :- ::ws.schema/workspace- "Atomically flip `workspace` to `status`, the first status of a run, unless- its current status is in `disallowed-statuses`. The conditional UPDATE is- the mutual-exclusion guard — two concurrent requests can never both start a- run. Throws a 400 when the current status disallows the run. Returns the- updated `workspace` copy."- [workspace :- ::ws.schema/workspace- status :- ::ws.schema/workspace-status- disallowed-statuses :- [:set ::ws.schema/workspace-status]]- (let [updated (t2/update! :model/Workspace- :id (:id workspace)- :status [:not-in disallowed-statuses]- {:status status, :status_details nil})]- (when (zero? updated)- (throw (ex-info "The workspace status does not allow starting this run"- {:status-code 400, :workspace-id (:id workspace)})))- (assoc workspace :status status, :status_details nil)))--(mu/defn set-initial-workspace-provisioning-status! :- ::ws.schema/workspace- "Atomically set `:database-provisioning`, the first status of the- provisioning path; throws a 400 when a run is already in flight or the- workspace is already `:provisioned`. Called before the background run- starts so the run is immediately visible as in flight. Returns the updated- `workspace` copy."- [workspace :- ::ws.schema/workspace]- (set-initial-workspace-status! workspace :database-provisioning- (conj ws.schema/in-flight-statuses :provisioned)))--(mu/defn set-initial-workspace-deprovisioning-status! :- ::ws.schema/workspace- "Atomically set `:instance-deprovisioning`, the first status of the- deprovisioning path; throws a 400 when a run is already in flight or the- workspace is already `:unprovisioned`. Called before the background run- starts so the run is immediately visible as in flight. Returns the updated- `workspace` copy."- [workspace :- ::ws.schema/workspace]- (set-initial-workspace-status! workspace :instance-deprovisioning- (conj ws.schema/in-flight-statuses :unprovisioned)))--(defn- workspace-databases- "The workspace's WorkspaceDatabase rows, in stable row order — the order the- database phases provision and deprovision them in."- [ws-id]- (t2/select :model/WorkspaceDatabase :workspace_id ws-id {:order-by [[:id :asc]]}))--(mu/defn- set-workspace-status! :- ::ws.schema/workspace- "Persist `status`/`status-details` on the row and return `workspace` with them- assoc'ed, so callers can keep threading the updated copy."- [workspace :- ::ws.schema/workspace- status :- ::ws.schema/workspace-status- status-details :- [:maybe :string]]- (t2/update! :model/Workspace (:id workspace) {:status status, :status_details status-details})- (assoc workspace :status status, :status_details status-details))--(mu/defn provision-workspace! :- ::ws.schema/workspace- "Provision `workspace` (blocking): every database, then the git branch, then- the child instance, ending `:provisioned`. Stops on the first failure — the- phase's `*-failure` status and the error message land on the workspace —- and rethrows. Retries skip work that already succeeded. Returns the updated- workspace copy."- [workspace :- ::ws.schema/workspace]- (as-> workspace ws- (set-workspace-status! ws :database-provisioning nil)- (try- (run! provisioning.database/provision-database! (workspace-databases (:id ws)))- ws- (catch Throwable t- (set-workspace-status! ws :database-provisioning-failure (ex-message t))- (throw t)))- (set-workspace-status! ws :branch-provisioning nil)- (try- (provisioning.remote-sync/provision-branch! ws)- (catch Throwable t- (set-workspace-status! ws :branch-provisioning-failure (ex-message t))- (throw t)))- (set-workspace-status! ws :instance-provisioning nil)- (try- (-> ws- provisioning.instance/deprovision-instance!- provisioning.instance/provision-instance!)- (catch Throwable t- (set-workspace-status! ws :instance-provisioning-failure (ex-message t))- (throw t)))- (set-workspace-status! ws :provisioned nil)))--(mu/defn deprovision-workspace! :- ::ws.schema/workspace- "Deprovision `workspace` (blocking): the child instance, then the git branch,- then every database, ending `:unprovisioned`. Stops on the first failure —- the phase's `*-failure` status and the error message land on the workspace —- and rethrows. Retries skip work that already succeeded. Returns the updated- workspace copy."- [workspace :- ::ws.schema/workspace]- (as-> workspace ws- (set-workspace-status! ws :instance-deprovisioning nil)- (try- (provisioning.instance/deprovision-instance! ws)- (catch Throwable t- (set-workspace-status! ws :instance-deprovisioning-failure (ex-message t))- (throw t)))- (set-workspace-status! ws :branch-deprovisioning nil)- (try- (provisioning.remote-sync/deprovision-branch! ws)- (catch Throwable t- (set-workspace-status! ws :branch-deprovisioning-failure (ex-message t))- (throw t)))- (set-workspace-status! ws :database-deprovisioning nil)- (try- (run! provisioning.database/deprovision-database! (workspace-databases (:id ws)))- ws- (catch Throwable t- (set-workspace-status! ws :database-deprovisioning-failure (ex-message t))- (throw t)))- (set-workspace-status! ws :unprovisioned nil)))+(comment metabase-enterprise.workspaces.models.workspace-database/keep-me)++;;; ------------------------------------------------- Protocol -------------------------------------------------------++(p/defprotocol+ Provisioner+ "Wrapper around driver workspace-isolation multimethods for testability.+ The default [[dispatching-provisioner]] delegates to the real driver multimethods.+ Tests can reify custom implementations that fail on demand, count calls, etc."+ (details [this driver database workspace]+ "Compute {:schema ... :database_details ...} for the workspace without touching+ the warehouse. Called before init! so destroy! can clean up a partial init.")+ (init! [this driver database workspace]+ "Create isolated schema + user. `workspace` carries the precomputed `:schema`+ and `:database_details` from [[details]].")+ (grant! [this driver database workspace schemas]+ "Grant read access on `schemas` to the workspace user/role. `schemas` is a+ vector of driver-opaque schema-name strings. 3-slot drivers (SQL Server,+ BigQuery) derive the catalog from `database.details`.")+ (destroy! [this driver database workspace]+ "Tear down isolated schema + user. Should be idempotent."))++(def dispatching-provisioner+ "Default Provisioner that dispatches to the driver multimethods.++ Each call is wrapped in [[driver.conn/with-admin-connection]] so the underlying+ driver impls acquire connections via the database's `:admin-details` overlay+ (when configured). Workspace DDL — `CREATE USER`, `CREATE SCHEMA`, `GRANT` —+ typically needs higher-privilege credentials than the regular query user, and+ the admin overlay is how operators provide them. `details` computes no DDL but+ still binds the overlay: some drivers derive connection strings/catalogs from+ the effective details."+ (reify Provisioner+ (details [_ driver database workspace]+ (driver.conn/with-admin-connection+ (driver/workspace-isolation-details driver database workspace)))+ (init! [_ driver database workspace]+ (driver.conn/with-admin-connection+ (driver/init-workspace-isolation! driver database workspace)))+ (grant! [_ driver database workspace schemas]+ (driver.conn/with-admin-connection+ (driver/grant-workspace-read-access! driver database workspace schemas)))+ (destroy! [_ driver database workspace]+ (driver.conn/with-admin-connection+ (driver/destroy-workspace-isolation! driver database workspace)))))++;;; ---------------------------------------------- Implementation ----------------------------------------------------++(def ^:private ^:const provisioning-lock-timeout-seconds+ "Max wait for another node/thread to finish provisioning the same row."+ 60)++(defn- wsd-lock-key+ "Per-row cluster-lock keyword."+ [workspace-database-id]+ (keyword "metabase-enterprise.workspaces.provisioning"+ (str "wsd-" workspace-database-id)))++(defn do-with-workspace-database-lock+ "Run `thunk` while holding the per-row cluster lock for `workspace-database-id`.+ [[provision-single!]]/[[deprovision-single!]] take this same lock internally, so+ the acquisition here is reentrant — it lets a caller atomically combine an app-db+ change with provisioning under a single lock, acquired *before* the row is+ mutated (the lock-before-mutate order the rest of the system relies on)."+ [workspace-database-id thunk]+ (cluster-lock/with-cluster-lock {:lock (wsd-lock-key workspace-database-id)+ :timeout-seconds provisioning-lock-timeout-seconds}+ (thunk)))++(defmacro with-workspace-database-lock+ "Sugar over [[do-with-workspace-database-lock]]."+ [workspace-database-id & body]+ `(do-with-workspace-database-lock ~workspace-database-id (fn [] ~@body)))++(defn- wsd-iso-workspace+ "The synthetic workspace map from which all warehouse identifiers for a+ WorkspaceDatabase row are derived. Every provision/deprovision/teardown path+ must build the exact same map — teardown recomputes what provision computed."+ [workspace-database-id]+ {:id workspace-database-id :name (str "wsd-" workspace-database-id)})++;;; ---------------------------------------- Single-database operations -----------------------------------------------++(defn provision-workspace-database!+ "Provision an isolated output schema and user for a `:provisioning`+ WorkspaceDatabase row. State transitions: `:provisioning` -> `:provisioned`+ on success, or back to `:unprovisioned` on failure."+ [workspace-database-id provisioner]+ (try+ (with-workspace-database-lock workspace-database-id+ (let [wsd (t2/select-one :model/WorkspaceDatabase :id workspace-database-id)]+ (when-not wsd+ (throw (ex-info "WorkspaceDatabase not found" {:id workspace-database-id})))+ (when-not (= :provisioning (:status wsd))+ (throw (ex-info "WorkspaceDatabase must be :provisioning to provision"+ {:id workspace-database-id :status (:status wsd)})))+ (let [db (t2/select-one :model/Database :id (:database_id wsd))+ driver (driver.u/database->driver db)+ workspace (wsd-iso-workspace workspace-database-id)+ ws-details (merge workspace (details provisioner driver db workspace))+ schemas (vec (:input_schemas wsd))]+ (try+ (init! provisioner driver db ws-details)+ (grant! provisioner driver db ws-details schemas)+ (catch Throwable t+ (try+ (destroy! provisioner driver db ws-details)+ (catch Throwable destroy-t+ (log/warnf destroy-t "Failed to clean up after provisioning failure for WorkspaceDatabase %s"+ workspace-database-id)))+ (throw t)))+ (t2/update! :model/WorkspaceDatabase+ {:id workspace-database-id}+ {:output_namespace (:schema ws-details)+ :database_details (:database_details ws-details)+ :status :provisioned}))))+ (catch Throwable t+ (t2/update! :model/WorkspaceDatabase+ {:id workspace-database-id}+ {:status :unprovisioned})+ (throw t)))+ (t2/select-one :model/WorkspaceDatabase :id workspace-database-id))++(defn deprovision-workspace-database!+ "Reverse provisioning for a `:deprovisioning` WorkspaceDatabase.+ State transitions: `:deprovisioning` -> `:unprovisioned` on success,+ or back to `:provisioned` on failure."+ [workspace-database-id provisioner]+ (try+ (with-workspace-database-lock workspace-database-id+ (let [wsd (t2/select-one :model/WorkspaceDatabase :id workspace-database-id)]+ (when-not wsd+ (throw (ex-info "WorkspaceDatabase not found" {:id workspace-database-id})))+ (when-not (= :deprovisioning (:status wsd))+ (throw (ex-info "WorkspaceDatabase must be :deprovisioning to deprovision"+ {:id workspace-database-id :status (:status wsd)})))+ (let [db (t2/select-one :model/Database :id (:database_id wsd))+ driver (driver.u/database->driver db)+ workspace (assoc (wsd-iso-workspace workspace-database-id)+ :schema (:output_namespace wsd)+ :database_details (:database_details wsd))]+ (try+ (destroy! provisioner driver db workspace)+ (finally+ ;; Clear `TableRemapping` rows whose `to_*` matches this workspace's+ ;; iso namespace. Runs in `finally` so a partial warehouse teardown+ ;; (e.g. BQ dataset deleted, SA delete throws) still leaves app-DB+ ;; in a "workspace no longer routes queries" state. Without this,+ ;; future queries against canonical tables would rewrite to an iso+ ;; namespace that no longer exists and 500 in the QP. The unique+ ;; constraint on `(database_id, from_*)` prevents two workspaces on+ ;; the same DB from remapping the same canonical table, so scoping+ ;; by iso namespace alone is correct.+ ;;+ ;; Rebind `*current-connectable*` to nil so the DELETE runs on a+ ;; fresh autocommit connection outside the workspace-database lock's+ ;; tx. Otherwise, when `destroy!` throws, the surrounding tx rolls+ ;; back and undoes the DELETE.+ (binding [t2.connection/*current-connectable* nil]+ (ws.remapping-cleanup/clear-mappings-for-iso! db+ (:database_id wsd)+ (:output_namespace wsd)))))+ (t2/update! :model/WorkspaceDatabase+ {:id workspace-database-id}+ {:output_namespace ""+ :database_details {}+ :status :unprovisioned}))))+ (catch Throwable t+ (t2/update! :model/WorkspaceDatabase+ {:id workspace-database-id}+ {:status :provisioned})+ (throw t)))+ (t2/select-one :model/WorkspaceDatabase :id workspace-database-id))++;;; ----------------------------------- High-level entry points (blocking) --------------------------------------------++(defn provision-single!+ "Flip a single WorkspaceDatabase to `:provisioning` and provision it synchronously.+ The row must be `:unprovisioned`. Returns the updated WorkspaceDatabase row."+ ([wsd-id]+ (provision-single! wsd-id dispatching-provisioner))+ ([wsd-id provisioner]+ (t2/update! :model/WorkspaceDatabase {:id wsd-id :status :unprovisioned}+ {:status :provisioning})+ (provision-workspace-database! wsd-id provisioner)))++(defn deprovision-single!+ "Flip a single WorkspaceDatabase to `:deprovisioning` and deprovision it synchronously.+ The row must be `:provisioned`. Returns the updated WorkspaceDatabase row."+ ([wsd-id]+ (deprovision-single! wsd-id dispatching-provisioner))+ ([wsd-id provisioner]+ (t2/update! :model/WorkspaceDatabase {:id wsd-id :status :provisioned}+ {:status :deprovisioning})+ (deprovision-workspace-database! wsd-id provisioner)))++(defn provision-workspace!+ "Flip every `:unprovisioned` WorkspaceDatabase under `workspace-id` to+ `:provisioning`, then provision each one synchronously (blocking).+ Returns the number of databases provisioned."+ ([workspace-id]+ (provision-workspace! workspace-id dispatching-provisioner))+ ([workspace-id provisioner]+ (let [triggered (t2/update! :model/WorkspaceDatabase+ {:workspace_id workspace-id :status :unprovisioned}+ {:status :provisioning})]+ (when (pos? triggered)+ (doseq [{:keys [id]} (t2/select [:model/WorkspaceDatabase :id]+ :workspace_id workspace-id+ :status :provisioning)]+ (try+ (provision-workspace-database! id provisioner)+ (catch Throwable t+ (log/warnf t "Failed to provision WorkspaceDatabase %s" id)))))+ triggered)))++(defn teardown-workspace-database!+ "Tear down one WorkspaceDatabase's warehouse isolation and, on success, delete+ its row. Used by the delete path and by create-workspace cleanup. Works from ANY+ state: the per-row cluster lock is taken first, so in-flight+ provisioning/deprovisioning on another node completes before we touch anything+ (or the lock times out and the teardown is reported as a failure), and the row+ is re-read under the lock so we see whatever that work persisted. Identifiers+ are the persisted ones, or+ recomputed deterministically from the row id via the provisioner's `details`+ ([[metabase.driver/workspace-isolation-details]]) for a crashed `:provisioning`+ row that never recorded anything.++ There is no partial outcome: the teardown either fully succeeds or the row+ stays. The row is the durable record that warehouse resources may exist — it+ must survive instance crashes and disappear only once the warehouse footprint+ is confirmed gone. On success the WorkspaceDatabase row is DELETED and the+ app-DB `TableRemapping` rows for the iso namespace are cleared — stale+ remappings would rewrite queries to a dropped schema and 500 the QP.++ Throws on failure. When the destroy itself fails, the row bookkeeping runs+ first: the row is kept, forced `:unprovisioned`, and the remapping cleanup+ still runs. Failures before the destroy — lock timeout, identifier+ computation — throw without touching the row. Either way the teardown can be+ retried. Returns nil."+ [wsd provisioner]+ (with-workspace-database-lock (:id wsd)+ (when-let [wsd (t2/select-one :model/WorkspaceDatabase :id (:id wsd))]+ (let [db (t2/select-one :model/Database :id (:database_id wsd))+ driver (driver.u/database->driver db)+ iso-ws (wsd-iso-workspace (:id wsd))+ computed (delay (details provisioner driver db iso-ws))+ schema (or (not-empty (:output_namespace wsd))+ (:schema @computed))+ db-details (if (seq (:database_details wsd))+ (:database_details wsd)+ (:database_details @computed))+ workspace (assoc iso-ws :schema schema :database_details db-details)]+ (try+ (destroy! provisioner driver db workspace)+ (catch Throwable t+ ;; The rethrow rolls back the cluster lock's wrapping transaction on+ ;; postgres/mysql app DBs, so the keep-row bookkeeping needs its own+ ;; autocommit connection to survive it.+ (binding [t2.connection/*current-connectable* nil]+ (t2/update! :model/WorkspaceDatabase {:id (:id wsd)}+ {:output_namespace "" :database_details {} :status :unprovisioned}))+ (throw t))+ (finally+ ;; App-DB cleanup needs no warehouse connection, so it ALWAYS runs — stale+ ;; remappings would otherwise rewrite queries to a dropped schema and 500 the+ ;; QP. Fresh autocommit connection to survive any surrounding tx rollback+ ;; (mirrors `deprovision-workspace-database!`).+ (binding [t2.connection/*current-connectable* nil]+ (ws.remapping-cleanup/clear-mappings-for-iso! db (:database_id wsd) schema))))+ (t2/delete! :model/WorkspaceDatabase :id (:id wsd))+ nil))))++(defn deprovision-workspace!+ "Flip every `:provisioned` WorkspaceDatabase under `workspace-id` to+ `:deprovisioning`, then deprovision each one synchronously (blocking).+ Returns the number of databases deprovisioned."+ ([workspace-id]+ (deprovision-workspace! workspace-id dispatching-provisioner))+ ([workspace-id provisioner]+ (let [triggered (t2/update! :model/WorkspaceDatabase+ {:workspace_id workspace-id :status :provisioned}+ {:status :deprovisioning})]+ (when (pos? triggered)+ (doseq [{:keys [id]} (t2/select [:model/WorkspaceDatabase :id]+ :workspace_id workspace-id+ :status :deprovisioning)]+ (try+ (deprovision-workspace-database! id provisioner)+ (catch Throwable t+ (log/warnf t "Failed to deprovision WorkspaceDatabase %s" id)))))+ triggered)))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/provisioning/database.clj b/enterprise/backend/src/metabase_enterprise/workspaces/provisioning/database.cljdeleted file mode 100644index 2f19afb9140..00000000000--- a/enterprise/backend/src/metabase_enterprise/workspaces/provisioning/database.clj+++ /dev/null@@ -1,225 +0,0 @@-(ns metabase-enterprise.workspaces.provisioning.database- "Warehouse-side provisioning for a single `:model/WorkspaceDatabase` row:- creating/destroying the isolated schema + user via the driver- workspace-isolation multimethods.-- Per-row lifecycle:-- provision- unprovisioned ────────────► provisioning ──────► provisioned- ▲ │ │- │ failure ▼ │ deprovision- │ provisioning-failure ▼- │ deprovisioning- │ │- │ failure ▼- └───────────────────────────◄──────── deprovisioning-failure (retry deprovision)-- Failures are recorded on the row (`:status_details` carries the error- message); there are no rollbacks or retries — both operations may simply be- invoked again from any status and no-op when there is nothing to do."- (:require- [metabase-enterprise.workspaces.models.table-remapping]- [metabase-enterprise.workspaces.models.workspace-database]- [metabase-enterprise.workspaces.schema :as ws.schema]- [metabase.driver :as driver]- [metabase.driver.connection :as driver.conn]- [metabase.driver.sql :as driver.sql]- [metabase.driver.util :as driver.u]- [metabase.util.malli :as mu]- [potemkin.types :as p]- [toucan2.core :as t2]))--(comment metabase-enterprise.workspaces.models.table-remapping/keep-me- metabase-enterprise.workspaces.models.workspace-database/keep-me)--;;; ------------------------------------------------- Protocol ---------------------------------------------------------(p/defprotocol+ DatabaseProvisioner- "Wrapper around driver workspace-isolation multimethods for testability.- The default [[database-provisioner]] delegates to the real driver multimethods.- Tests can reify custom implementations that fail on demand, count calls, etc."- (details [this driver database workspace]- "Compute {:schema ... :database_details ...} for the workspace without touching- the warehouse. Called before init! so destroy! can clean up a partial init.")- (init! [this driver database workspace]- "Create isolated schema + user. `workspace` carries the precomputed `:schema`- and `:database_details` from [[details]].")- (grant! [this driver database workspace schemas]- "Grant read access on `schemas` to the workspace user/role. `schemas` is a- vector of driver-opaque schema-name strings. 3-slot drivers (SQL Server,- BigQuery) derive the catalog from `database.details`.")- (destroy! [this driver database workspace]- "Tear down isolated schema + user. Should be idempotent."))--(def database-provisioner- "Default DatabaseProvisioner that dispatches to the driver multimethods.-- Each call is wrapped in [[driver.conn/with-admin-connection]] so the underlying- driver impls acquire connections via the database's `:admin-details` overlay- (when configured). Workspace DDL — `CREATE USER`, `CREATE SCHEMA`, `GRANT` —- typically needs higher-privilege credentials than the regular query user, and- the admin overlay is how operators provide them. `details` computes no DDL but- still binds the overlay: some drivers derive connection strings/catalogs from- the effective details."- (reify DatabaseProvisioner- (details [_ driver database workspace]- (driver.conn/with-admin-connection- (driver/workspace-isolation-details driver database workspace)))- (init! [_ driver database workspace]- (driver.conn/with-admin-connection- (driver/init-workspace-isolation! driver database workspace)))- (grant! [_ driver database workspace schemas]- (driver.conn/with-admin-connection- (driver/grant-workspace-read-access! driver database workspace schemas)))- (destroy! [_ driver database workspace]- (driver.conn/with-admin-connection- (driver/destroy-workspace-isolation! driver database workspace)))))--;;; -------------------------------------------- Engine namespaces -----------------------------------------------------(mu/defn engine-namespace-positions :- [:map [:db [:maybe :string]] [:schema [:maybe :string]]]- "Return `{:db ?, :schema ?}` — the values that should populate the `:db` and- `:schema` AST slots for a Table row in `database`. `table` is optional; pass- it when you want the schema position derived from the Table's `:schema`- column (the normal `spec-for-table` case). Pass nil for `table` when you only- need the `:db` slot (workspace `output_namespace` expansion, GRANT emission).-- `nil` for either slot means \"this driver doesn't emit this AST level.\"- Empty-string sentinel coercion happens at the storage boundary, not here.-- Driven by [[metabase.driver.sql/table-qualification-style]] +- [[metabase.driver.sql/db-slot-value]] -- third-party drivers participate by- implementing those rather than getting a new case branch here."- ([database] (engine-namespace-positions database nil))- ([database :- :map table :- [:maybe :map]]- (case (driver.sql/table-qualification-style (:engine database))- :table-qualification-style/table- {:db nil- :schema nil}-- :table-qualification-style/schema-table- {:db nil- :schema (:schema table)}-- :table-qualification-style/db-table- {:db (driver.sql/db-slot-value (:engine database) database)- :schema nil}-- :table-qualification-style/db-schema-table- {:db (driver.sql/db-slot-value (:engine database) database)- :schema (:schema table)})))--;;; ---------------------------------------------- Implementation ------------------------------------------------------(defn- wsd-iso-workspace- "The synthetic workspace map from which all warehouse identifiers for a- WorkspaceDatabase row are derived. Every provision/deprovision path must build- the exact same map — deprovision recomputes what provision computed."- [wsd-id]- {:id wsd-id :name (str "wsd-" wsd-id)})--(defn- iso-db-slot- "Value of the `:db` AST slot a `TableRemapping.to_db` carries for `database`'s- engine. Empty string for drivers that don't populate the `:db` slot- (Postgres, Redshift, H2, ClickHouse) — the storage sentinel. For drivers- that do (MySQL, SQL Server, BigQuery), consults- [[metabase.driver.sql/db-slot-value]]."- [database]- (or (driver.sql/db-slot-value (:engine database) database) ""))--(defn clear-mappings-for-iso!- "Delete every `TableRemapping` row on `database`'s id whose `to_*` slots match- the iso namespace `(iso-db, iso-schema)` for this workspace_database row.-- Idempotent — 0 rows deleted is a valid outcome when nothing was registered- (e.g. a workspace_database that was provisioned but never had a transform run).-- Scope rationale: the unique constraint on `(database_id, from_db, from_schema,- from_table_name)` prevents two workspaces on the same metabase_database from- remapping the same canonical table. So deleting by iso namespace is enough- to avoid clobbering another workspace's rows on the same database_id.-- Returns the count of rows deleted."- [database database-id output-namespace]- (long- (t2/delete! :model/TableRemapping- :database_id database-id- :to_db (iso-db-slot database)- :to_schema (or output-namespace ""))))--(mu/defn provision-database! :- :nil- "Provision an isolated output schema and user for one WorkspaceDatabase row- (blocking). No-op when the row is already `:provisioned`, so retries are safe.- State transitions: `:provisioning` -> `:provisioned` on success, or- `:provisioning-failure` with the error message in `:status_details` on failure- — no rollback, the error is rethrown and a later retry picks up from whatever- state the warehouse is in."- ([workspace-database]- (provision-database! workspace-database database-provisioner))- ([{wsd-id :id :as wsd} :- ::ws.schema/workspace-database- provisioner]- (when-not (= :provisioned (:status wsd))- (t2/update! :model/WorkspaceDatabase wsd-id {:status :provisioning, :status_details nil})- (try- (let [db (t2/select-one :model/Database :id (:database_id wsd))- driver (driver.u/database->driver db)- workspace (wsd-iso-workspace wsd-id)- ws-details (merge workspace (details provisioner driver db workspace))]- (init! provisioner driver db ws-details)- (grant! provisioner driver db ws-details (vec (:input_schemas wsd)))- (t2/update! :model/WorkspaceDatabase wsd-id- {:output_namespace (:schema ws-details)- :database_details (:database_details ws-details)- :status :provisioned}))- (catch Throwable t- (t2/update! :model/WorkspaceDatabase wsd-id- {:status :provisioning-failure, :status_details (ex-message t)})- (throw t))))- nil))--(mu/defn deprovision-database! :- :nil- "Tear down the warehouse isolation of one WorkspaceDatabase row (blocking).- No-op when the row is already `:unprovisioned`, so retries are safe. Works- from ANY other state: warehouse identifiers are the persisted ones, or- recomputed deterministically from the row id via the provisioner's `details`- for a crashed `:provisioning` row that never recorded anything.-- State transitions: `:deprovisioning` -> `:unprovisioned` on success (the- persisted identifiers are cleared), or `:deprovisioning-failure` with the- error message in `:status_details` on failure — the error is rethrown and the- deprovision can be retried.-- App-DB `TableRemapping` rows for the row's iso namespace are ALWAYS cleared,- even when the warehouse teardown fails partway — stale remappings would- rewrite queries to a dropped schema and 500 the QP."- ([workspace-database]- (deprovision-database! workspace-database database-provisioner))- ([{wsd-id :id :as wsd} :- ::ws.schema/workspace-database- provisioner]- (when-not (= :unprovisioned (:status wsd))- (t2/update! :model/WorkspaceDatabase wsd-id {:status :deprovisioning, :status_details nil})- (try- (let [db (t2/select-one :model/Database :id (:database_id wsd))- driver (driver.u/database->driver db)- iso-ws (wsd-iso-workspace wsd-id)- computed (delay (details provisioner driver db iso-ws))- schema (or (not-empty (:output_namespace wsd))- (:schema @computed))- db-details (if (seq (:database_details wsd))- (:database_details wsd)- (:database_details @computed))- workspace (assoc iso-ws :schema schema :database_details db-details)]- (try- (destroy! provisioner driver db workspace)- (finally- (clear-mappings-for-iso! db (:database_id wsd) schema)))- (t2/update! :model/WorkspaceDatabase wsd-id- {:output_namespace ""- :database_details {}- :status :unprovisioned}))- (catch Throwable t- (t2/update! :model/WorkspaceDatabase wsd-id- {:status :deprovisioning-failure, :status_details (ex-message t)})- (throw t))))- nil))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/provisioning/instance.clj b/enterprise/backend/src/metabase_enterprise/workspaces/provisioning/instance.cljdeleted file mode 100644index 5177c33bee2..00000000000--- a/enterprise/backend/src/metabase_enterprise/workspaces/provisioning/instance.clj+++ /dev/null@@ -1,128 +0,0 @@-(ns metabase-enterprise.workspaces.provisioning.instance- "Child-instance provisioning for a workspace: creating/destroying the- developer Metabase instance a workspace's users work in, and persisting its- identifiers (`instance_id`/`instance_url`) on the `:model/Workspace` row.-- The default [[instance-provisioner]] is a stub — the real (Harbormaster)- implementation comes later."- (:require- [metabase-enterprise.workspaces.config :as ws.config]- [metabase-enterprise.workspaces.schema :as ws.schema]- [metabase.util.jvm :as u.jvm]- [metabase.util.malli :as mu]- [potemkin.types :as p]- [toucan2.core :as t2]))--(set! *warn-on-reflection* true)--(p/defprotocol+ InstanceProvisioner- "Provisioning operations for a workspace's child Metabase instance.- The default [[instance-provisioner]] is a stub; tests reify custom- implementations that fail on demand, count calls, etc."- (create! [this workspace config]- "Start creating a child instance for `workspace`, booted from the- config-file map `config` (see- [[metabase-enterprise.workspaces.config/build-workspace-config]]).- Creation is asynchronous: returns `::ws.schema/instance` whose `:status`- may still be `:creating` — the instance is not usable until [[fetch]]- reports `:active`.")- (fetch [this instance-id]- "Fetch a child instance by id. Returns `::ws.schema/instance`.")- (delete! [this instance-id]- "Delete the child instance with `instance-id`. Should be idempotent."))--(def instance-provisioner- "Default InstanceProvisioner. Stub implementation for now: pretends a child- instance was created by returning a random id and a placeholder url that- are immediately `:active`."- (reify InstanceProvisioner- (create! [_ _workspace _config]- {:id (str (random-uuid)), :url "https://example.com", :status :active})- (fetch [_ instance-id]- {:id instance-id, :url "https://example.com", :status :active})- (delete! [_ _instance-id]- nil)))--(def instance-poll-interval-ms- "How often [[provision-instance!]] polls the provisioner while the child- instance is still coming up."- 5000)--(def instance-poll-timeout-ms- "How long [[provision-instance!]] waits for the child instance to reach a- terminal status before giving up."- (* 10 60 1000))--(mu/defn- wait-for-instance :- ::ws.schema/instance- "Poll `provisioner` until `instance-id` reaches a terminal status and return- the instance. Throws when it lands anywhere but `:active`, or when- [[instance-poll-timeout-ms]] elapses first."- [provisioner instance-id :- :string]- (let [{:keys [status] :as instance}- (u.jvm/poll {:thunk #(fetch provisioner instance-id)- :done? #(contains? #{:active :error} (:status %))- :interval-ms instance-poll-interval-ms- :timeout-ms instance-poll-timeout-ms})]- (when-not instance- (throw (ex-info "Timed out waiting for the workspace instance to start"- {:instance_id instance-id, :timeout-ms instance-poll-timeout-ms})))- (when-not (= :active status)- (throw (ex-info "Workspace instance failed to start"- {:instance_id instance-id, :status status})))- instance))--(mu/defn- create-instance! :- ::ws.schema/workspace- "Create a fresh child instance for `workspace` (blocking). Persists- `:instance_id` (clearing any stale url) as soon as the provisioner accepts- the creation, then polls until the instance is `:active` and persists the- url the active instance reports. Throws when the instance lands in `:error`- or the startup times out. Returns `workspace` with the new instance fields- assoc'ed."- [workspace :- ::ws.schema/workspace- provisioner]- (let [config (-> workspace- (t2/hydrate :databases)- ws.config/build-workspace-config)- {:keys [id]} (create! provisioner workspace config)]- (t2/update! :model/Workspace (:id workspace) {:instance_id id, :instance_url nil})- (let [url (:url (wait-for-instance provisioner id))]- (t2/update! :model/Workspace (:id workspace) {:instance_url url})- (assoc workspace :instance_id id, :instance_url url))))--(mu/defn- delete-instance! :- ::ws.schema/workspace- "Delete `workspace`'s child instance (blocking) and clear its- `:instance_id`/`:instance_url`. Throws on failure — the caller records the- failure on the workspace status. Returns `workspace` with the instance- fields cleared."- [workspace :- ::ws.schema/workspace- provisioner]- (delete! provisioner (:instance_id workspace))- (t2/update! :model/Workspace (:id workspace) {:instance_id nil, :instance_url nil})- (assoc workspace :instance_id nil, :instance_url nil))--(mu/defn provision-instance! :- ::ws.schema/workspace- "Provision a fresh child instance for `workspace` (blocking) — callers- deprovision any existing one first, so every run starts from a clean state.- Throws on failure or startup timeout — the caller records the failure on- the workspace status. Returns `workspace` with the new instance fields- assoc'ed."- ([workspace]- (provision-instance! workspace instance-provisioner))- ([workspace :- ::ws.schema/workspace- provisioner]- (create-instance! workspace provisioner)))--(mu/defn deprovision-instance! :- ::ws.schema/workspace- "Delete `workspace`'s child instance (blocking) and clear its- `:instance_id`/`:instance_url`. No-op when the workspace has no instance, so- retries are safe. Throws on failure — the caller records the failure on the- workspace status. Returns `workspace` with the instance fields cleared."- ([workspace]- (deprovision-instance! workspace instance-provisioner))- ([workspace :- ::ws.schema/workspace- provisioner]- (if-not (:instance_id workspace)- (do- (t2/update! :model/Workspace (:id workspace) {:instance_url nil})- (assoc workspace :instance_url nil))- (delete-instance! workspace provisioner))))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/provisioning/remote_sync.clj b/enterprise/backend/src/metabase_enterprise/workspaces/provisioning/remote_sync.cljdeleted file mode 100644index cc5713e8b5f..00000000000--- a/enterprise/backend/src/metabase_enterprise/workspaces/provisioning/remote_sync.clj+++ /dev/null@@ -1,40 +0,0 @@-(ns metabase-enterprise.workspaces.provisioning.remote-sync- "Git-branch provisioning for workspaces. A workspace with a `:target_branch`- owns a branch on the remote-sync repo: provisioning creates it (idempotent —- an existing branch is fine), deprovisioning deletes it. `:target_branch`- itself never changes. Both phases are no-ops when remote sync is disabled- or the workspace has no branch."- (:require- [clojure.string :as str]- [metabase-enterprise.remote-sync.core :as remote-sync]- [metabase-enterprise.workspaces.schema :as ws.schema]- [metabase.util.malli :as mu]))--(set! *warn-on-reflection* true)--(defn- workspace-branch [workspace]- (let [branch (:target_branch workspace)]- (when-not (str/blank? branch)- branch)))--(mu/defn provision-branch! :- ::ws.schema/workspace- "Create the workspace's `:target_branch` on the remote-sync repo. A no-op- when remote sync is disabled, the workspace has no branch, or the branch- already exists. Returns `workspace` unchanged."- [workspace :- ::ws.schema/workspace]- (when-let [branch (workspace-branch workspace)]- (when (and (remote-sync/remote-sync-enabled)- (not (remote-sync/branch-exists? branch)))- (remote-sync/create-branch! branch)))- workspace)--(mu/defn deprovision-branch! :- ::ws.schema/workspace- "Delete the workspace's `:target_branch` from the remote-sync repo. A no-op- when remote sync is disabled at this point or the workspace has no branch.- `:target_branch` itself is kept as is — a later provision recreates the- branch. Returns `workspace` unchanged."- [workspace :- ::ws.schema/workspace]- (when-let [branch (workspace-branch workspace)]- (when (remote-sync/remote-sync-enabled)- (remote-sync/delete-branch! branch)))- workspace)diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/remapping_cleanup.clj b/enterprise/backend/src/metabase_enterprise/workspaces/remapping_cleanup.cljnew file mode 100644index 00000000000..f6521f56240--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/workspaces/remapping_cleanup.clj@@ -0,0 +1,45 @@+(ns metabase-enterprise.workspaces.remapping-cleanup+ "Lifecycle hook: clear `TableRemapping` rows when a `WorkspaceDatabase` is+ deprovisioned.++ Lives as a leaf to break a `provisioning` -> `table-remapping` -> `core` ->+ `provisioning` require cycle: the deprovision path needs to delete remapping+ rows scoped to the workspace's iso namespace, but `table-remapping` reaches+ back through `workspaces.core` which loads `provisioning`. The `:db`-slot+ derivation goes through [[metabase.driver.sql/db-slot-value]] so this ns+ has no workspace deps beyond the model registration."+ (:require+ [metabase-enterprise.workspaces.models.table-remapping]+ [metabase.driver.sql :as driver.sql]+ [toucan2.core :as t2]))++(comment metabase-enterprise.workspaces.models.table-remapping/keep-me)++(defn- iso-db-slot+ "Value of the `:db` AST slot a `TableRemapping.to_db` carries for `database`'s+ engine. Empty string for drivers that don't populate the `:db` slot+ (Postgres, Redshift, H2, ClickHouse) — the storage sentinel. For drivers+ that do (MySQL, SQL Server, BigQuery), consults+ [[metabase.driver.sql/db-slot-value]]."+ [database]+ (or (driver.sql/db-slot-value (:engine database) database) ""))++(defn clear-mappings-for-iso!+ "Delete every `TableRemapping` row on `database`'s id whose `to_*` slots match+ the iso namespace `(iso-db, iso-schema)` for this workspace_database row.++ Idempotent — 0 rows deleted is a valid outcome when nothing was registered+ (e.g. a workspace_database that was provisioned but never had a transform run).++ Scope rationale: the unique constraint on `(database_id, from_db, from_schema,+ from_table_name)` prevents two workspaces on the same metabase_database from+ remapping the same canonical table. So deleting by iso namespace is enough+ to avoid clobbering another workspace's rows on the same database_id.++ Returns the count of rows deleted."+ [database database-id output-namespace]+ (long+ (t2/delete! :model/TableRemapping+ :database_id database-id+ :to_db (iso-db-slot database)+ :to_schema (or output-namespace ""))))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/schema.clj b/enterprise/backend/src/metabase_enterprise/workspaces/schema.cljdeleted file mode 100644index 0830623ceea..00000000000--- a/enterprise/backend/src/metabase_enterprise/workspaces/schema.clj+++ /dev/null@@ -1,93 +0,0 @@-(ns metabase-enterprise.workspaces.schema- "Malli schemas shared across the workspaces module."- (:require- [metabase.util.malli.registry :as mr]- [metabase.util.malli.schema :as ms]))--(mr/def ::workspace-status- "Lifecycle status of a `:model/Workspace`. Deliberately kept separate from- `::workspace-database-status` — the workspace lifecycle spans more phases- than a single database's.-- Provisioning path (databases, then the git branch, then the child instance):-- unprovisioned ──► database-provisioning ──► branch-provisioning ──► instance-provisioning ──► provisioned- │ │ │- failure ▼ failure ▼ failure ▼- database-provisioning-failure branch-provisioning-failure instance-provisioning-failure-- Deprovisioning path (the child instance, then the git branch, then databases):-- provisioned ──► instance-deprovisioning ──► branch-deprovisioning ──► database-deprovisioning ──► unprovisioned- │ │ │- failure ▼ failure ▼ failure ▼- instance-deprovisioning-failure branch-deprovisioning-failure database-deprovisioning-failure-- `/provision` may be started from any settled status except `:provisioned`;- `/deprovision` from any settled status except `:unprovisioned` (both 400- otherwise, including while a run is in flight). The database phases skip- rows that already reached the target state; the branch phases are no-ops- when remote sync is disabled or the workspace has no `:target_branch`; the- instance phase of a provision always deletes and recreates the child- instance."- [:enum- :unprovisioned- :database-provisioning- :database-provisioning-failure- :branch-provisioning- :branch-provisioning-failure- :instance-provisioning- :instance-provisioning-failure- :provisioned- :instance-deprovisioning- :instance-deprovisioning-failure- :branch-deprovisioning- :branch-deprovisioning-failure- :database-deprovisioning- :database-deprovisioning-failure])--(def in-flight-statuses- "The workspace statuses that mark a provision or deprovision run as currently- executing. All other statuses are settled — a new run may start from them."- #{:database-provisioning :branch-provisioning :instance-provisioning- :instance-deprovisioning :branch-deprovisioning :database-deprovisioning})--(mr/def ::instance-status- "Status of a workspace's child instance: `:creating` — still coming up, keep- polling; `:active` — up and usable; `:error` — terminal, stop waiting."- [:enum :creating :active :error])--(mr/def ::instance- "A workspace's child instance as reported by an InstanceProvisioner. The url- is only set once the instance is `:active`."- [:map- [:id :string]- [:url [:maybe :string]]- [:status ::instance-status]])--(mr/def ::workspace-database-status- "Lifecycle status of a `:model/WorkspaceDatabase`. The `*-failure` statuses are- terminal until the next provision/deprovision retry."- [:enum- :unprovisioned- :provisioning- :provisioning-failure- :provisioned- :deprovisioning- :deprovisioning-failure])--(mr/def ::workspace- "A `:model/Workspace` row."- [:map- [:id ms/PositiveInt]- [:name ms/NonBlankString]- [:status {:optional true} ::workspace-status]- [:status_details {:optional true} [:maybe :string]]- [:instance_id {:optional true} [:maybe :string]]- [:instance_url {:optional true} [:maybe :string]]])--(mr/def ::workspace-database- "A `:model/WorkspaceDatabase` row."- [:map- [:id ms/PositiveInt]- [:status {:optional true} ::workspace-database-status]])diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/settings.clj b/enterprise/backend/src/metabase_enterprise/workspaces/settings.cljindex 7ced87b5973..7df12766529 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/settings.clj@@ -3,6 +3,10 @@[metabase.settings.core :refer [defsetting]][metabase.util.i18n :refer [deferred-tru]]))+(def keep-me+ "Marker so callers can `(comment ...keep-me)` to retain the require that registers the settings in this ns."+ nil)+(defsetting database-enable-workspaces(deferred-tru "Whether to enable Workspaces for a specific Database."):default false@@ -13,18 +17,8 @@:export? false:database-local :only)-(defsetting workspace-instance-user-password- (deferred-tru "Default password for the workspace creator''s superuser account on workspace child instances.")- :type :string- :visibility :admin- :encryption :when-encryption-key-set- :sensitive? true- :audit :never- :export? false- :doc false)-(defsetting instance-workspace- (deferred-tru "The workspace loaded on this instance. Populated at boot from a config.yml `:workspace` section or at runtime via `POST /api/ee/advanced-config`. `nil` on parent and unconfigured instances. Read by the QP, transform hooks, and the EE `workspace-mode?` predicate via `metabase-enterprise.workspaces.instance/instance-workspace`.")+ (deferred-tru "The workspace loaded on this instance. Populated at boot from a config.yml `:workspace` section or at runtime via `POST /api/ee/advanced-config`. `nil` on parent and unconfigured instances. Read by the QP, transform hooks, and the EE `workspace-mode?` predicate via `metabase-enterprise.workspaces.core/instance-workspace`."):type :json:encryption :no:visibility :internaldiff --git a/enterprise/backend/src/metabase_enterprise/workspaces/table_remapping.clj b/enterprise/backend/src/metabase_enterprise/workspaces/table_remapping.cljindex 300bb0a6063..77f746a546b 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/table_remapping.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/table_remapping.clj@@ -46,9 +46,8 @@right slots from a `:model/Database` and `:model/Table` row."(:require[clojure.set :as set]- [metabase-enterprise.workspaces.instance :as ws.instance]+ [metabase-enterprise.workspaces.core :as ws][metabase-enterprise.workspaces.models.table-remapping]- [metabase-enterprise.workspaces.provisioning.database :as provisioning.database][metabase-enterprise.workspaces.remapping.core :as ws.remapping][metabase.driver :as driver][metabase.driver.connection :as driver.conn]@@ -87,10 +86,10 @@(defn db-position-value"Value to put in the `:db` slot for `database`. Convenience accessor over- [[provisioning.database/engine-namespace-positions]]. Only meaningful for- drivers whose `qualified-name-components` includes `:db`."+ [[ws/engine-namespace-positions]]. Only meaningful for drivers whose+ `qualified-name-components` includes `:db`."[database]- (:db (provisioning.database/engine-namespace-positions database)))+ (:db (ws/engine-namespace-positions database)))(mu/defn spec-for-table :- ::table-spec"Return `{:db, :schema, :table}` for `table` in `database`, populating only the@@ -112,7 +111,7 @@[database :- [:map [:engine :keyword]]table :- [:map [:name :string]]](let [components (set (driver/qualified-name-components (:engine database)))- positions (provisioning.database/engine-namespace-positions database table)]+ positions (ws/engine-namespace-positions database table)]{:db (normalize-level (when (:db components) (:db positions))):schema (normalize-level (when (:schema components) (:schema positions))):table (:name table)}))@@ -326,8 +325,8 @@to derive any missing slot from the `database`, so the driver-aware match in`canonical->isolated` doesn't false-miss on a populated remap row."[database from-spec]- (let [positions (provisioning.database/engine-namespace-positions database {:name (:name from-spec)- :schema (:schema from-spec)})]+ (let [positions (ws/engine-namespace-positions database {:name (:name from-spec)+ :schema (:schema from-spec)})](cond-> from-spec(nil? (:db from-spec)) (assoc :db (:db positions))(nil? (:schema from-spec)) (assoc :schema (:schema positions)))))@@ -613,7 +612,7 @@- From-side: a synthetic `:model/Database` + `:model/Table` shape is fed through`spec-for-table` so `:db` and `:schema` slots are filled per the driver's`qualified-name-components`.- - To-side: workspace output namespace comes from `ws.instance/db-workspace-namespace`,+ - To-side: workspace output namespace comes from `ws/db-workspace-namespace`,a `{:db ?, :schema ?}` map sourced from `config.yml`. The `:db` and`:schema` slots both flow into the `TableRemapping` row's `to_db` /`to_schema` columns - so cross-DB SQL Server / BigQuery workspaces are@@ -625,9 +624,9 @@can be `assoc`ed onto a transform target without a per-call denormalization shim.Throws when the database is not workspaced -- reaching this with a non-workspaced- db is a programming error; gate on [[ws.instance/db-workspace-namespace]] first."+ db is a programming error; gate on [[ws/db-workspace-namespace]] first."[db-id target]- (let [workspace-ns (ws.instance/db-workspace-namespace db-id)]+ (let [workspace-ns (ws/db-workspace-namespace db-id)](when-not workspace-ns(throw (ex-info "Cannot record transform-target remapping: database is not workspaced"{:db-id db-id :target target})))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/transform_hooks.clj b/enterprise/backend/src/metabase_enterprise/workspaces/transform_hooks.cljindex 39d00267eb0..4e4b6dfa905 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/transform_hooks.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/transform_hooks.clj@@ -1,7 +1,7 @@(ns metabase-enterprise.workspaces.transform-hooks"Enterprise hooks that integrate the transform execution path with workspace isolation."(:require- [metabase-enterprise.workspaces.instance :as ws.instance]+ [metabase-enterprise.workspaces.core :as ws][metabase-enterprise.workspaces.remapping.core :as ws.remapping][metabase-enterprise.workspaces.table-remapping :as ws.table-remapping][metabase.premium-features.core :refer [defenterprise]]))@@ -34,7 +34,7 @@provisioned, the rewrite must apply.":feature :none[db-id target]- (if (some? (ws.instance/db-workspace-namespace db-id))+ (if (some? (ws/db-workspace-namespace db-id))(let [{to-db :db to-schema :schema to-name :name}(ws.table-remapping/add-transform-target-mapping! db-id target)];; **Replace** `:db` and `:schema` on the target with the to-spec's valuesdiff --git a/enterprise/backend/test/metabase_enterprise/advanced_config/api_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_config/api_test.cljindex 0c388014554..247b4997f14 100644--- a/enterprise/backend/test/metabase_enterprise/advanced_config/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/advanced_config/api_test.clj@@ -1,13 +1,10 @@(ns metabase-enterprise.advanced-config.api-test(:require[clojure.test :refer [deftest is testing use-fixtures]]- [metabase-enterprise.advanced-config.file :as advanced-config.file][metabase-enterprise.workspaces.core :as ws][metabase.driver.util]- [metabase.setup.core :as setup][metabase.test :as mt][metabase.test.fixtures :as fixtures]- [metabase.util :as u][metabase.util.yaml :as yaml][toucan2.core :as t2]))@@ -18,13 +15,9 @@(defn- yaml-bytes ^bytes [body](.getBytes ^String (yaml/generate-string body) "UTF-8"))-(defn- multipart- ([bs]- (multipart bs nil))- ([bs api-key]- [{:request-options {:headers (cond-> {"content-type" "multipart/form-data"}- api-key (assoc "x-metabase-apikey" api-key))}}- {:config bs}]))+(defn- multipart [bs]+ [{:request-options {:headers {"content-type" "multipart/form-data"}}}+ {:config bs}])(deftest superuser-only-test(testing "POST /api/ee/advanced-config requires superuser"@@ -32,25 +25,6 @@(apply mt/user-http-request :rasta :post 403 "ee/advanced-config/"(multipart (yaml-bytes {:version 1 :config {}})))))))-(deftest external-requires-static-api-key-test- (testing "POST /api/ee/advanced-config/external"- (let [payload (yaml-bytes {:version 1 :config {}})]- (testing "403 when MB_API_KEY is not set"- (mt/with-temporary-setting-values [api-key nil]- (apply mt/client :post 403 "ee/advanced-config/external"- (multipart payload "anything"))))- (mt/with-temporary-setting-values [api-key "test-api-key"]- (testing "403 without the header"- (apply mt/client :post 403 "ee/advanced-config/external"- (multipart payload)))- (testing "403 with a wrong key"- (apply mt/client :post 403 "ee/advanced-config/external"- (multipart payload "wrong-key")))- (testing "applies the config with the correct key, no user session needed"- (mt/with-premium-features #{:config-text-file}- (apply mt/client :post 204 "ee/advanced-config/external"- (multipart payload "test-api-key"))))))))-(deftest does-not-expand-templates-test(testing "POST /api/ee/advanced-config does NOT expand `{{env VAR}}` templates";; The literal template appears in the inserted database's name; if env@@ -102,6 +76,34 @@(when-let [db-id (t2/select-one-pk :model/Database :name db-name :engine "postgres")](t2/delete! :model/Database db-id)))))))+(deftest runtime-upload-via-api-does-not-set-the-lock-test+ (testing "POST /api/ee/advanced-config with a :workspace section does NOT lock the workspace (only the boot loader does)"+ (let [db-name (str "advanced-config-ws-" (random-uuid))+ payload {:version 1+ :config {:databases [{:name db-name+ :engine "postgres"+ :details {}}]+ :workspace {:name "Uploaded Workspace"+ :databases {(keyword db-name) {:input_schemas ["public"]+ :output {:schema "ws_uploaded"}}}}}}+ lock-atom @#'ws/locked-by-config?*+ prior @lock-atom]+ (try+ (reset! lock-atom false)+ (mt/with-premium-features #{:config-text-file :workspaces}+ (mt/with-temporary-setting-values [config-from-file-sync-databases false]+ (with-redefs [metabase.driver.util/can-connect-with-details? (constantly true)]+ (mt/user-http-request :crowberto :post 204 "ee/advanced-config/"+ (first (multipart (yaml-bytes payload)))+ (second (multipart (yaml-bytes payload)))))))+ (is (false? (ws/workspace-locked-by-config?))+ "a runtime upload must not flip the boot lock")+ (finally+ (reset! lock-atom prior)+ (ws/clear-instance-workspace!)+ (when-let [db-id (t2/select-one-pk :model/Database :name db-name :engine "postgres")]+ (t2/delete! :model/Database db-id)))))))+(deftest workspace-section-rejected-without-workspaces-feature-test(testing "POST /api/ee/advanced-config returns 402 when :workspace section requires :workspaces"(let [payload {:version 1@@ -114,51 +116,3 @@(mt/user-http-request :crowberto :post 402 "ee/advanced-config/"(first (multipart (yaml-bytes payload)))(second (multipart (yaml-bytes payload)))))))))--(deftest config-upload-user-entry-test- (testing "POST /api/ee/advanced-config with a users entry creates the user"- (mt/with-premium-features #{:config-text-file}- (mt/with-model-cleanup [:model/User]- (let [email (u/lower-case-en (str (mt/random-name) "@config.test"))]- (apply mt/user-http-request :crowberto :post 204 "ee/advanced-config/"- (multipart (yaml-bytes {:version 1- :config {:users [{:first_name "Work"- :last_name "Space"- :email email- :password "sUp3r-s3cret1"- :is_superuser true}]}})))- (is (=? {:is_superuser true}- (t2/select-one :model/User :email email))- "the user from the config lands as a superuser"))))))--(deftest config-user-entry-completes-setup-test- (testing "a users entry completes the initial setup on an empty instance (has-user-setup flips)"- (mt/with-empty-h2-app-db!- (mt/with-premium-features #{:config-text-file}- (binding [advanced-config.file/*supported-versions* {:min 1 :max 1}]- (is (false? (setup/has-user-setup)))- (advanced-config.file/initialize!- {:version 1- :config {:users [{:first_name "First"- :last_name "Last"- :email "[email protected]"- :password "sUp3r-s3cret1"- :is_superuser true}]}})- (is (true? (setup/has-user-setup)))- (is (=? {:is_superuser true}- (t2/select-one :model/User :email "[email protected]"))))))))--(deftest config-user-entry-allows-nil-names-test- (testing "users entries without a first or last name are allowed"- (mt/with-premium-features #{:config-text-file}- (mt/with-model-cleanup [:model/User]- (let [email (u/lower-case-en (str (mt/random-name) "@config.test"))]- (binding [advanced-config.file/*supported-versions* {:min 1 :max 1}]- (advanced-config.file/initialize!- {:version 1- :config {:users [{:first_name nil- :last_name nil- :email email- :password "sUp3r-s3cret1"}]}}))- (is (=? {:first_name nil, :last_name nil}- (t2/select-one :model/User :email email))))))))diff --git a/enterprise/backend/test/metabase_enterprise/advanced_config/file/databases_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_config/file/databases_test.cljindex b8820227ac3..01fb1020f6e 100644--- a/enterprise/backend/test/metabase_enterprise/advanced_config/file/databases_test.clj+++ b/enterprise/backend/test/metabase_enterprise/advanced_config/file/databases_test.clj@@ -9,7 +9,7 @@[metabase.sync.sync-metadata :as sync-metadata][metabase.test :as mt][metabase.util :as u]- [metabase.util.jvm :as u.jvm]+ [metabase.util.quick-task :as quick-task][toucan2.core :as t2])(:import(clojure.lang ExceptionInfo)))@@ -84,7 +84,7 @@(mt/with-temporary-setting-values [config-from-file-sync-databases true](try(let [submit-calls (atom 0)]- (with-redefs [u.jvm/run-in-virtual-thread (fn [_] (swap! submit-calls inc))]+ (with-redefs [quick-task/submit-task! (fn [_] (swap! submit-calls inc))](testing "config entry with :is_stub true and empty :details is accepted"(is (= :ok(advanced-config.file/initialize!diff --git a/enterprise/backend/test/metabase_enterprise/advanced_config/file/workspace_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_config/file/workspace_test.cljindex 31481cfa5b6..9e37f4ff24b 100644--- a/enterprise/backend/test/metabase_enterprise/advanced_config/file/workspace_test.clj+++ b/enterprise/backend/test/metabase_enterprise/advanced_config/file/workspace_test.clj@@ -5,11 +5,9 @@[clojure.test :refer [deftest is testing use-fixtures]][metabase-enterprise.advanced-config.file :as advanced-config.file][metabase-enterprise.advanced-config.file.workspace :as advanced-config.file.workspace]- [metabase-enterprise.remote-sync.core :as remote-sync]- [metabase-enterprise.workspaces.instance :as ws.instance]+ [metabase-enterprise.workspaces.core :as ws][metabase.test :as mt][metabase.test.fixtures :as fixtures]- [metabase.util.jvm :as u.jvm][metabase.util.yaml :as yaml])(:import(clojure.lang ExceptionInfo)))@@ -21,12 +19,17 @@;; Several tests drive a `:workspace` section through `initialize!`, which;; requires the `:workspaces` feature; default it on here. Tests that assert;; the feature's *absence* override with their own `mt/with-premium-features`.+ ;; Also save/restore the boot-lock atom so the boot-lock tests below can't+ ;; leak a flipped lock into the rest of the suite.(mt/with-premium-features #{:workspaces}(binding [advanced-config.file/*supported-versions* {:min 1, :max 1}]- (try- (thunk)- (finally- (ws.instance/clear-instance-workspace!)))))))+ (let [lock-atom @#'ws/locked-by-config?*+ prior @lock-atom]+ (try+ (thunk)+ (finally+ (ws/clear-instance-workspace!)+ (reset! lock-atom prior))))))))(defn- load-fixture-by-driver [driver](-> (str "metabase_enterprise/workspaces/resources/workspace_config_" (name driver) ".yml")@@ -84,7 +87,7 @@(is (= "New workspace" workspace-name))(is (= 1 database-count))(testing "setting holds the parsed config"- (let [stored (ws.instance/instance-workspace)]+ (let [stored (ws/instance-workspace)](is (= "New workspace" (:name stored)))(is (= ["public"](get-in stored [:databases db-id :input_schemas])))@@ -102,12 +105,12 @@:databases {:ws-test-db-2 {:input_schemas ["public"]:output {:schema "different_schema"}}})](advanced-config.file.workspace/apply-workspace-section! section-1)- (is (= "New workspace" (:name (ws.instance/instance-workspace))))+ (is (= "New workspace" (:name (ws/instance-workspace))))(advanced-config.file.workspace/apply-workspace-section! section-2)- (is (= "Renamed Workspace" (:name (ws.instance/instance-workspace))))- (is (= 1 (count (:databases (ws.instance/instance-workspace)))))+ (is (= "Renamed Workspace" (:name (ws/instance-workspace))))+ (is (= 1 (count (:databases (ws/instance-workspace)))))(is (= {:schema "different_schema"}- (->> (ws.instance/instance-workspace) :databases vals first :output))+ (->> (ws/instance-workspace) :databases vals first :output))"setting reflects the new config, not the old one"))))))(deftest unknown-database-name-throws-test@@ -125,7 +128,7 @@(mt/with-temp [:model/Database {db-id :id} {:name "ws-test-db" :engine :postgres}](advanced-config.file.workspace/apply-workspace-section!(workspace-section "ws-test-db"))- (is (= {:schema "mb__isolation_44490_1933"} (ws.instance/db-workspace-namespace db-id)))))))+ (is (= {:schema "mb__isolation_44490_1933"} (ws/db-workspace-namespace db-id)))))))(deftest db-workspace-namespace-returns-nil-when-no-load-test(testing "without a config.yml load, db-workspace-namespace returns nil — manager-side rows are not consulted"@@ -133,7 +136,7 @@;; the read truly comes from the setting and not from any leftover rows.(mt/with-empty-h2-app-db!(mt/with-temp [:model/Database {db-id :id} {:name "ws-test-db" :engine :postgres}]- (is (nil? (ws.instance/db-workspace-namespace db-id)))))))+ (is (nil? (ws/db-workspace-namespace db-id)))))))(deftest oss-rejects-workspace-section-without-premium-token-test(testing "OSS instances need the :config-text-file token to load a :workspace section (no special bring-up carve-out)"@@ -179,10 +182,10 @@(advanced-config.file/initialize!{:version 1:config {:workspace (workspace-section "ws-test-db")}})- (is (= "New workspace" (:name (ws.instance/instance-workspace)))+ (is (= "New workspace" (:name (ws/instance-workspace)))"the workspace section should have been applied")(is (= {:schema "mb__isolation_44490_1933"}- (ws.instance/db-workspace-namespace db-id))))))))+ (ws/db-workspace-namespace db-id))))))))(deftest workspaces-gate-is-scoped-to-workspace-section-test(testing "without :workspaces, other non-:workspace sections still load (gate is :workspace-scoped)"@@ -192,7 +195,7 @@{:version 1:config {:users [{:first_name "X" :last_name "Y":email "[email protected]" :password "pw"}]}})- (is (nil? (ws.instance/instance-workspace))+ (is (nil? (ws/instance-workspace))"no workspace section means no workspace state was set")))));;; ----------------------------------------- per-driver shapes -----------------------------------------@@ -209,7 +212,7 @@{:name "ws":databases {:mysql-prod {:input_schemas ["prod_db"]:output {:db "ws_alice"}}}})- (is (= {:db "ws_alice"} (ws.instance/db-workspace-namespace db-id))))))))+ (is (= {:db "ws_alice"} (ws/db-workspace-namespace db-id))))))))(deftest bigquery-3-slot-section-test(testing "BigQuery workspace: project + dataset in the :output map"@@ -219,7 +222,7 @@{:name "ws":databases {:bq-prod {:input_schemas ["core"]:output {:db "metabase-prod" :schema "ws_alice"}}}})- (is (= {:db "metabase-prod" :schema "ws_alice"} (ws.instance/db-workspace-namespace db-id)))))))+ (is (= {:db "metabase-prod" :schema "ws_alice"} (ws/db-workspace-namespace db-id)))))));;; ----------------------------------------- per-driver YAML fixtures -----------------------------------------;;;@@ -295,47 +298,54 @@:engine (:engine expectations):details (:details expectations)}](advanced-config.file.workspace/apply-workspace-section! section)- (is (= (:output expectations) (ws.instance/db-workspace-namespace db-id))+ (is (= (:output expectations) (ws/db-workspace-namespace db-id))(str driver " setting output matches fixture"))))))))))-(deftest workspace-with-remote-settings-starts-import-test- (testing "a config with both a :workspace section and remote-sync settings starts the initial import"- (mt/with-premium-features #{:workspaces :config-text-file}- (mt/with-temp [:model/Database _ {:name "ws-test-db" :engine :postgres}]- (mt/with-temporary-setting-values [remote-sync-url nil- remote-sync-branch nil- remote-sync-token nil]- (let [started (atom 0)]- (with-redefs [u.jvm/run-in-virtual-thread (fn [thunk] (thunk) nil)- remote-sync/start-import! (fn [] (swap! started inc) nil)]- (advanced-config.file/initialize!- {:version 1- :config {:settings {:remote-sync-url "https://git.example.com/test.git"- :remote-sync-branch "main"- :remote-sync-token "s3cr3t"}- :workspace (workspace-section "ws-test-db")}})- (is (= 1 @started)))))))- (testing "but not without remote-sync settings"- (mt/with-premium-features #{:workspaces :config-text-file}+;;; ----------------------------------------- boot lock wiring ----------------------------------------+;;;+;;; `boot-initialize!` is the sole boot entry point. When it parses a+;;; `config.yml` containing a `:workspace` section, it flips the in-memory+;;; `workspace-locked-by-config?*` atom. Runtime callers of `initialize!` (the+;;; `POST /api/ee/advanced-config` path) and direct callers of+;;; `apply-workspace-section!` do NOT flip the lock — the boot wrapper is the+;;; only thing that does.++(defn- lock-atom [] @#'ws/locked-by-config?*)++(deftest boot-initialize!-sets-the-lock-when-config-has-workspace-test+ (testing "boot-initialize!, given a parsed config.yml with a :workspace section, flips the lock"+ (mt/with-empty-h2-app-db!+ (mt/with-premium-features #{:config-text-file :workspaces}(mt/with-temp [:model/Database _ {:name "ws-test-db" :engine :postgres}]- (let [started (atom 0)]- (with-redefs [u.jvm/run-in-virtual-thread (fn [thunk] (thunk) nil)- remote-sync/start-import! (fn [] (swap! started inc) nil)]- (advanced-config.file/initialize!- {:version 1- :config {:workspace (workspace-section "ws-test-db")}})- (is (zero? @started)))))))- (testing "and not without a :workspace section"- (mt/with-premium-features #{:workspaces :config-text-file}- (mt/with-temporary-setting-values [remote-sync-url nil- remote-sync-branch nil- remote-sync-token nil]- (let [started (atom 0)]- (with-redefs [u.jvm/run-in-virtual-thread (fn [thunk] (thunk) nil)- remote-sync/start-import! (fn [] (swap! started inc) nil)]- (advanced-config.file/initialize!- {:version 1- :config {:settings {:remote-sync-url "https://git.example.com/test.git"- :remote-sync-branch "main"- :remote-sync-token "s3cr3t"}}})- (is (zero? @started)))))))))+ (reset! (lock-atom) false)+ (with-redefs [advanced-config.file/config-from-disk+ (constantly {:version 1+ :config {:workspace (workspace-section "ws-test-db")}})]+ (advanced-config.file/boot-initialize!))+ (is (true? (ws/workspace-locked-by-config?))))))))++(deftest boot-initialize!-does-not-set-the-lock-when-config-has-no-workspace-test+ (testing "boot-initialize! with no :workspace section in the config leaves the lock false"+ (mt/with-empty-h2-app-db!+ (reset! (lock-atom) false)+ (with-redefs [advanced-config.file/config-from-disk+ (constantly {:version 1+ :config {:settings {}}})]+ (advanced-config.file/boot-initialize!))+ (is (false? (ws/workspace-locked-by-config?))))))++(deftest boot-initialize!-no-config-file-does-not-set-the-lock-test+ (testing "boot-initialize! when no config.yml is present (config-from-disk returns nil) leaves the lock false"+ (mt/with-empty-h2-app-db!+ (reset! (lock-atom) false)+ (with-redefs [advanced-config.file/config-from-disk (constantly nil)]+ (advanced-config.file/boot-initialize!))+ (is (false? (ws/workspace-locked-by-config?))))))++(deftest apply-workspace-section!-direct-call-does-not-set-the-lock-test+ (testing "calling apply-workspace-section! directly (the runtime / test path) does NOT flip the lock"+ (mt/with-empty-h2-app-db!+ (mt/with-temp [:model/Database _ {:name "ws-test-db" :engine :postgres}]+ (reset! (lock-atom) false)+ (advanced-config.file.workspace/apply-workspace-section! (workspace-section "ws-test-db"))+ (is (false? (ws/workspace-locked-by-config?)))))))diff --git a/enterprise/backend/test/metabase_enterprise/checker/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/checker/e2e_test.cljdeleted file mode 100644index 880e5ba4fe9..00000000000--- a/enterprise/backend/test/metabase_enterprise/checker/e2e_test.clj+++ /dev/null@@ -1,137 +0,0 @@-(ns metabase-enterprise.checker.e2e-test- "End-to-end test running the checker against the serialization baseline export.-- This exercises the full pipeline: YAML parsing → index building → store →- provider → query validation → structural checks → results.-- The baseline at test_resources/serialization_baseline is a real serdes export- with cards, dashboards, transforms, snippets, and documents backed by- the sample database schema."- (:require- [clojure.string :as str]- [clojure.test :refer :all]- [clojure.walk :as walk]- [metabase-enterprise.checker.semantic :as checker]))--(set! *warn-on-reflection* true)--(def ^:private export-dir "test_resources/serialization_baseline")-(def ^:private schema-dir "test_resources/serialization_baseline/databases")--(def ^:private baseline-results- "Checker results for the baseline export. Computed once, shared across all tests."- (delay (:results (checker/check export-dir schema-dir))))--(defn- failures-report- "Format failing results for assertion messages."- [results status]- (->> results- (filter #(= status (checker/result-status (second %))))- (map checker/format-error)- (str/join "\n")))--(deftest baseline-export-no-errors-test- (testing "The serialization baseline export passes all checks with zero errors"- (let [results @baseline-results- summary (checker/summarize-results results)]- (is (=? {:ok pos-int?- :errors 0- :unresolved 0- :native-errors 0- :issues 0}- summary))- (is (> (:total summary) 45)- "Should check cards plus non-card entities (dashboards, transforms, etc.)")- (is (pos? (:total summary))- "Should check at least one entity")- (is (zero? (:errors summary))- (str "Errors:\n" (failures-report results :error)))- (is (zero? (:unresolved summary))- (str "Unresolved:\n" (failures-report results :unresolved)))- (is (zero? (:native-errors summary))- (str "Native SQL errors:\n" (failures-report results :native-errors)))- (is (zero? (:issues summary))- (str "Issues:\n" (failures-report results :issues))))))--(deftest baseline-export-card-refs-populated-test- (testing "Checked cards have populated ref data (tables, fields)"- (let [card-results (->> @baseline-results vals (filter :refs))]- (is (pos? (count card-results))- "At least some cards should have refs")- (is (some #(seq (get-in % [:refs :tables])) card-results)- "At least one card should reference tables")- (is (some #(seq (get-in % [:refs :fields])) card-results)- "At least one card should reference fields"))))--;;; ===========================================================================-;;; Card shape assertions — verify detailed analysis on specific cards-;;; ===========================================================================--(defn- normalize [x] (walk/postwalk x (cond-> x (sequential? x) set)))--(deftest joins-card-shape-test- (testing "MBQL card with 3 joins resolves all tables and fields"- (let [result (get @baseline-results "wpW9JO5MsHQd0-7WrR7gN")]- (is (=? (normalize- {:card-id pos-int?- :name "Joins - Strategies and Compound Conditions",- :entity-id "wpW9JO5MsHQd0-7WrR7gN",- :refs {:tables ["Sample Database.PUBLIC.PRODUCTS"- "Sample Database.PUBLIC.REVIEWS"- "Sample Database.PUBLIC.ORDERS"- "Sample Database.PUBLIC.PEOPLE"],- :fields ["Sample Database.PUBLIC.ORDERS.ID"- "Sample Database.PUBLIC.ORDERS.USER_ID"- "Sample Database.PUBLIC.ORDERS.PRODUCT_ID"- "Sample Database.PUBLIC.ORDERS.SUBTOTAL"- "Sample Database.PUBLIC.ORDERS.TAX"- "Sample Database.PUBLIC.ORDERS.TOTAL"- "Sample Database.PUBLIC.ORDERS.DISCOUNT"- "Sample Database.PUBLIC.ORDERS.CREATED_AT"- "Sample Database.PUBLIC.ORDERS.QUANTITY"- "Sample Database.PUBLIC.PRODUCTS.ID"- "Sample Database.PUBLIC.PRODUCTS.EAN"- "Sample Database.PUBLIC.PRODUCTS.TITLE"- "Sample Database.PUBLIC.PRODUCTS.CATEGORY"- "Sample Database.PUBLIC.PRODUCTS.VENDOR"- "Sample Database.PUBLIC.PRODUCTS.PRICE"- "Sample Database.PUBLIC.PRODUCTS.RATING"- "Sample Database.PUBLIC.PRODUCTS.CREATED_AT"- "Sample Database.PUBLIC.PEOPLE.NAME"- "Sample Database.PUBLIC.PEOPLE.STATE"]}})- (normalize result)))- ;; REVIEWS fields are NOT in the list because the join has `fields: none`- ;; — only join condition fields from REVIEWS are used, not selected.- ;; The REVIEWS table IS resolved though (checked above).- (is (-> result :source-cards empty?)))))--;;;; note: these are all sets here for equality's sake. But they traditionally are vectors--(deftest measure-metric-segment-refs-test- (testing "Card with measure, metric, and segment references tracks them all"- (let [result (get @baseline-results "kBjQ5VXJ5z3vYSW72J6qa")]- (is (=? {:card-id pos-int?,- :name "Metric, Measure, and Segment References",- :entity-id "kBjQ5VXJ5z3vYSW72J6qa",- :refs {:tables ["Sample Database.PUBLIC.ORDERS"],- :source-cards ["Total Revenue"],- :measures ["DBap2523KuN4Lt-cYrUjF"],- :metrics ["IW8kbqZVaMxdGtCM2F4U6"],- :segments ["OaFXAvEzPvu9ZXW5PUMRa"]}}- result)))))--(deftest transitive-entity-refs-test- (testing "Source card reference transitively surfaces the inner card's snippets, measures, segments"- (let [;; "Native Query - Card and Snippet References" sources from- ;; "Basic Aggregations" AND uses a snippet.- snippet-card (get @baseline-results "HiBFSt0BNx5s5MxVDLLKB")]- ;; The snippet card itself uses a snippet and sources from Basic Aggregations.- ;; Basic Aggregations references ORDERS table — that should appear transitively.- (is (=? {:card-id pos-int?- :name "Native Query - Card and Snippet References",- :entity-id "HiBFSt0BNx5s5MxVDLLKB",- :refs {:source-cards ["Basic Aggregations"],- :tables ["Sample Database.PUBLIC.ORDERS"],- :fields ["Sample Database.PUBLIC.ORDERS.CREATED_AT"],- :snippets ["Active Product Filter"]}}- snippet-card)))))diff --git a/enterprise/backend/test/metabase_enterprise/checker/format/concise_schema_test.clj b/enterprise/backend/test/metabase_enterprise/checker/format/concise_schema_test.cljdeleted file mode 100644index 4767123fbb6..00000000000--- a/enterprise/backend/test/metabase_enterprise/checker/format/concise_schema_test.clj+++ /dev/null@@ -1,119 +0,0 @@-(ns metabase-enterprise.checker.format.concise-schema-test- "Tests for the concise JSON schema format — single file with three flat lists."- (:require- [clojure.java.io :as io]- [clojure.test :refer :all]- [metabase-enterprise.checker.format.concise-schema :as concise-schema]- [metabase-enterprise.checker.source :as source]))--(set! *warn-on-reflection* true)--(def ^:private fixture-file "test_resources/yaml_checks/concise_metadata.json")--(defn- fixture-path []- (let [f (io/file fixture-file)]- (if (.isAbsolute f)- fixture-file- (.getPath (io/file (System/getProperty "user.dir") fixture-file)))))--;;; ===========================================================================-;;; Database resolution-;;; ===========================================================================--(deftest resolve-database-test- (let [source (concise-schema/make-source (fixture-path))]- (testing "resolves known database"- (let [db (source/resolve-database source "Test Database")]- (is (some? db))- (is (= "Test Database" (:name db)))- (is (= "h2" (:engine db)))))- (testing "returns nil for unknown database"- (is (nil? (source/resolve-database source "Nonexistent"))))))--(deftest all-database-names-test- (let [source (concise-schema/make-source (fixture-path))]- (is (= #{"Test Database" "SQLite DB"}- (set (source/all-database-names source))))))--;;; ===========================================================================-;;; Table resolution-;;; ===========================================================================--(deftest resolve-table-test- (let [source (concise-schema/make-source (fixture-path))]- (testing "resolves table with schema"- (let [table (source/resolve-table source ["Test Database" "public" "orders"])]- (is (some? table))- (is (= "orders" (:name table)))- (is (= "public" (:schema table)))))- (testing "resolves schema-less table"- (let [table (source/resolve-table source ["SQLite DB" nil "orders"])]- (is (some? table))- (is (= "orders" (:name table)))))- (testing "returns nil for unknown table"- (is (nil? (source/resolve-table source ["Test Database" "public" "nope"]))))))--(deftest all-table-paths-test- (let [source (concise-schema/make-source (fixture-path))- tables (source/all-table-paths source)]- (is (= 4 (count tables)))- (is (some #(= ["Test Database" "public" "orders"] %) tables))- (is (some #(= ["Test Database" "public" "products"] %) tables))- (is (some #(= ["SQLite DB" nil "orders"] %) tables))- (is (some #(= ["SQLite DB" nil "users"] %) tables))))--(deftest tables-for-database-test- (let [source (concise-schema/make-source (fixture-path))]- (is (= 2 (count (source/tables-for-database source "Test Database"))))- (is (= 2 (count (source/tables-for-database source "SQLite DB"))))))--;;; ===========================================================================-;;; Field resolution-;;; ===========================================================================--(deftest resolve-field-test- (let [source (concise-schema/make-source (fixture-path))]- (testing "resolves field with schema"- (let [field (source/resolve-field source ["Test Database" "public" "orders" "total"])]- (is (some? field))- (is (= "total" (:name field)))- (is (= "type/Float" (:base_type field)))- (is (= "DOUBLE PRECISION" (:database_type field)))))- (testing "resolves schema-less field"- (let [field (source/resolve-field source ["SQLite DB" nil "orders" "id"])]- (is (some? field))- (is (= "id" (:name field)))))- (testing "returns nil for unknown field"- (is (nil? (source/resolve-field source ["Test Database" "public" "orders" "nope"]))))))--(deftest fields-for-table-test- (let [source (concise-schema/make-source (fixture-path))]- (testing "returns field paths for table"- (let [fields (source/fields-for-table source ["Test Database" "public" "orders"])]- (is (set? fields))- (is (= 4 (count fields)))- (is (contains? fields ["Test Database" "public" "orders" "id"]))- (is (contains? fields ["Test Database" "public" "orders" "total"]))))- (testing "returns nil for unknown table"- (is (nil? (source/fields-for-table source ["Nope" "x" "y"]))))))--(deftest all-field-paths-test- (let [source (concise-schema/make-source (fixture-path))]- (is (= 11 (count (source/all-field-paths source))))))--;;; ===========================================================================-;;; Data shape — returned maps have keys the provider expects-;;; ===========================================================================--(deftest table-has-schema-and-db-id-test- (testing "table data includes :schema and :db_id for provider"- (let [source (concise-schema/make-source (fixture-path))- table (source/resolve-table source ["Test Database" "public" "orders"])]- (is (= "public" (:schema table)))- (is (= "Test Database" (:db_id table))))))--(deftest field-has-table-id-test- (testing "field data includes :table_id as path vector for provider"- (let [source (concise-schema/make-source (fixture-path))- field (source/resolve-field source ["Test Database" "public" "orders" "total"])]- (is (= ["Test Database" "public" "orders"] (:table_id field))))))diff --git a/enterprise/backend/test/metabase_enterprise/checker/format/serdes_assets_test.clj b/enterprise/backend/test/metabase_enterprise/checker/format/serdes_assets_test.cljdeleted file mode 100644index 55fc8b3e21c..00000000000--- a/enterprise/backend/test/metabase_enterprise/checker/format/serdes_assets_test.clj+++ /dev/null@@ -1,178 +0,0 @@-(ns metabase-enterprise.checker.format.serdes-assets-test- "Tests for the serdes assets format module — YAML extraction, file indexing, and assets source resolution."- (:require- [clojure.java.io :as io]- [clojure.test :refer [deftest is testing]]- [metabase-enterprise.checker.format.serdes-assets :as serdes]- [metabase-enterprise.checker.source :as source]- [metabase.util.yaml :as yaml]))--(set! *warn-on-reflection* true)--;;; ===========================================================================-;;; Helpers-;;; ===========================================================================--(def ^:private fixtures-dir "test_resources/yaml_checks")--(defn- fixtures-path []- (let [f (io/file fixtures-dir)]- (if (.isAbsolute f)- fixtures-dir- (.getPath (io/file (System/getProperty "user.dir") fixtures-dir)))))--(defn- with-temp-dir- "Create a temp directory, call (f dir-path), then clean up."- [f]- (let [dir (java.io.File/createTempFile "serdes-test" "")]- (.delete dir)- (.mkdirs dir)- (try- (f (.getPath dir))- (finally- (doseq [fl (reverse (file-seq dir))]- (.delete ^java.io.File fl))))))--(defn- write-yaml!- "Write a YAML file, creating parent directories as needed."- [dir & path-and-data]- (let [data (last path-and-data)- path-segs (butlast path-and-data)- ^java.io.File file (apply io/file dir (map str path-segs))]- (.mkdirs (.getParentFile file))- (spit file (yaml/generate-string data))- (.getPath file)))--;;; ===========================================================================-;;; YAML extraction tests (pure, no files needed)-;;; ===========================================================================--(deftest extract-entity-id-from-fixture-test- (testing "extract-entity-id reads entity_id from a card YAML"- (let [card-file (io/file (fixtures-path) "collections" "cards" "simple-orders_simple_orders.yaml")]- (is (= "simple-orders" (serdes/extract-entity-id (.getPath card-file)))))))--(deftest extract-model-from-fixture-test- (testing "extract-model reads the serdes model from a card YAML"- (let [card-file (io/file (fixtures-path) "collections" "cards" "simple-orders_simple_orders.yaml")]- (is (= "Card" (serdes/extract-model (.getPath card-file)))))))--(deftest extract-entity-id-from-temp-file-test- (testing "extract-entity-id with various formats"- (with-temp-dir- (fn [dir]- (let [f (write-yaml! dir "test.yaml" {:name "X" :entity_id "xYz123AbC"})]- (is (= "xYz123AbC" (serdes/extract-entity-id f))))))))--;;; ===========================================================================-;;; Index tests — databases are indexed, tables/fields are on-demand-;;; ===========================================================================--(deftest build-file-index-databases-test- (testing "build-file-index indexes databases from fixtures"- (let [index (serdes/build-file-index (fixtures-path))]- (is (contains? (:database index) "Test Database"))- (is (contains? (:database index) "SQLite DB")))))--(deftest build-file-index-cards-test- (testing "build-file-index indexes cards by entity_id"- (let [index (serdes/build-file-index (fixtures-path))]- (is (contains? (:card index) "simple-orders"))- (is (contains? (:card index) "native-orders"))- (is (contains? (:card index) "orders-with-products")))))--(deftest build-file-index-no-duplicates-test- (testing "fixture index has no duplicates"- (let [index (serdes/build-file-index (fixtures-path))]- (is (nil? (:duplicates index))))))--;;; ===========================================================================-;;; AssetsSource protocol — resolution tests-;;; ===========================================================================--(deftest source-resolves-card-test- (testing "SerdesSource resolves cards by entity-id"- (let [source (serdes/make-source (fixtures-path))- card (source/resolve-card source "simple-orders")]- (is (some? card))- (is (= "Simple Orders" (:name card))))))--(deftest source-returns-nil-for-unknown-test- (testing "SerdesSource returns nil for unknown entities"- (let [source (serdes/make-source (fixtures-path))]- (is (nil? (source/resolve-card source "no-such-card"))))))--;;; ===========================================================================-;;; Segments and measures — indexed from export dir-;;; ===========================================================================--(deftest export-dir-indexes-segments-and-measures-test- (testing "build-file-index (export dir) finds segments and measures under databases/"- (let [index (serdes/build-file-index (fixtures-path))]- (is (contains? (:segment index) "big-orders-segment")- "segment from with-schema table should be indexed")- (is (contains? (:segment index) "recent-orders-segment")- "segment from schema-less table should be indexed")- (is (contains? (:measure index) "total-revenue-measure")- "measure should be indexed"))))--(deftest segments-resolve-from-export-source-test- (testing "segments can be resolved from the export source"- (let [source (serdes/make-source (fixtures-path))]- (let [seg (source/resolve-segment source "big-orders-segment")]- (is (some? seg))- (is (= "Big Orders" (:name seg))))- (let [seg (source/resolve-segment source "recent-orders-segment")]- (is (some? seg))- (is (= "Recent Orders" (:name seg)))))))--(deftest measures-resolve-from-export-source-test- (testing "measures can be resolved from the export source"- (let [source (serdes/make-source (fixtures-path))- m (source/resolve-measure source "total-revenue-measure")]- (is (some? m))- (is (= "Total Revenue" (:name m))))))--;;; ===========================================================================-;;; Temp file edge cases-;;; ===========================================================================--(deftest build-file-index-empty-dir-test- (testing "build-file-index on empty directory returns empty index"- (with-temp-dir- (fn [dir]- (let [index (serdes/build-file-index dir)]- (is (nil? (:database index)))- (is (nil? (:card index))))))))--(def ^:private card-yaml-template- "name: %s\nentity_id: %s\nserdes/meta:\n- id: %s\n model: Card\n")--(deftest build-file-index-detects-duplicates-test- (testing "build-file-index reports duplicate entity_ids"- (with-temp-dir- (fn [dir]- ;; Create two cards with the same entity_id in different directories- (let [^java.io.File f1 (io/file dir "collections" "a" "cards" "dup.yaml")- ^java.io.File f2 (io/file dir "collections" "b" "cards" "dup.yaml")]- (.mkdirs (.getParentFile f1))- (.mkdirs (.getParentFile f2))- (spit f1 (format card-yaml-template "Card A" "dup-id" "dup-id"))- (spit f2 (format card-yaml-template "Card B" "dup-id" "dup-id"))- (let [index (serdes/build-file-index dir)]- (is (seq (:duplicates index)) "Should detect duplicate entity_ids")- (is (= "dup-id" (:ref (first (:duplicates index)))))))))))--(deftest source-index-accessor-test- (testing "source-index returns the assets index (no databases — those are in the schema source)"- (let [source (serdes/make-source (fixtures-path))- index (serdes/source-index source)]- (is (map? index))- (is (not (contains? index :database)) "databases are in the schema source, not the assets index")- (is (contains? index :card)))))--(deftest all-card-ids-test- (testing "all-card-ids returns all card entity-ids"- (let [source (serdes/make-source (fixtures-path))]- (is (>= (count (serdes/all-card-ids source)) 5))- (is (contains? (set (serdes/all-card-ids source)) "simple-orders")))))diff --git a/enterprise/backend/test/metabase_enterprise/checker/format/serdes_schema_test.clj b/enterprise/backend/test/metabase_enterprise/checker/format/serdes_schema_test.cljdeleted file mode 100644index 9cc7b1d0941..00000000000--- a/enterprise/backend/test/metabase_enterprise/checker/format/serdes_schema_test.clj+++ /dev/null@@ -1,262 +0,0 @@-(ns metabase-enterprise.checker.format.serdes-schema-test- "Tests for the serdes schema format module — database/table/field resolution and lazy indexing."- (:require- [clojure.java.io :as io]- [clojure.test :refer :all]- [metabase-enterprise.checker.format.serdes-schema :as serdes-schema]- [metabase-enterprise.checker.source :as source]- [metabase.util.yaml :as yaml]))--(set! *warn-on-reflection* true)--;;; ===========================================================================-;;; Helpers-;;; ===========================================================================--(def ^:private fixtures-dir "test_resources/yaml_checks")--(defn- fixtures-path []- (let [f (io/file fixtures-dir)]- (if (.isAbsolute f)- fixtures-dir- (.getPath (io/file (System/getProperty "user.dir") fixtures-dir)))))--(defn- with-temp-dir- "Create a temp directory, call (f dir-path), then clean up."- [f]- (let [dir (java.io.File/createTempFile "serdes-test" "")]- (.delete dir)- (.mkdirs dir)- (try- (f (.getPath dir))- (finally- (doseq [fl (reverse (file-seq dir))]- (.delete ^java.io.File fl))))))--(defn- write-yaml!- "Write a YAML file, creating parent directories as needed."- [dir & path-and-data]- (let [data (last path-and-data)- path-segs (butlast path-and-data)- ^java.io.File file (apply io/file dir (map str path-segs))]- (.mkdirs (.getParentFile file))- (spit file (yaml/generate-string data))- (.getPath file)))--(deftest build-database-dir-index-test- (testing "build-database-dir-index indexes databases and provides db-name->dir mapping"- (let [db-dir (str (fixtures-path) "/databases")- {:keys [index db-name->dir]} (serdes-schema/build-database-dir-index db-dir)]- (is (contains? (:database index) "Test Database"))- (is (contains? (:database index) "SQLite DB"))- (is (contains? db-name->dir "Test Database"))- (is (contains? db-name->dir "SQLite DB"))- ;; Tables are NOT in the index — they're resolved on demand- (is (nil? (:table index)))- ;; Should not contain cards (databases dir has no collections)- (is (nil? (:card index))))))--;;; ===========================================================================-;;; db-name->dir mapping tests-;;; ===========================================================================--(deftest db-name-to-dir-mapping-test- (testing "db-name->dir maps real database names to directory names"- (with-temp-dir- (fn [dir]- ;; Create a database with a slugified directory name but real name in YAML- (write-yaml! dir "databases" "my_fancy_db" "my_fancy_db.yaml"- {:name "My Fancy DB" :engine "h2"})- (let [{:keys [db-name->dir]} (serdes-schema/build-database-dir-index (str dir "/databases"))]- (is (= "my_fancy_db" (get db-name->dir "My Fancy DB")))))))- (testing "db-name->dir includes ALL databases, not just some"- (with-temp-dir- (fn [dir]- (doseq [i (range 10)]- (let [slug (str "db_" i)]- (write-yaml! dir "databases" slug (str slug ".yaml")- {:name (str "Database " i) :engine "h2"})))- (let [{:keys [db-name->dir]} (serdes-schema/build-database-dir-index (str dir "/databases"))]- (is (= 10 (count db-name->dir)))- (doseq [i (range 10)]- (is (= (str "db_" i) (get db-name->dir (str "Database " i))))))))))--;;; ===========================================================================-;;; SchemaSource protocol — on-demand resolution tests-;;; ===========================================================================--(deftest source-resolves-database-test- (testing "SerdesSchemaSource resolves databases by name"- (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))- db (source/resolve-database source "Test Database")]- (is (some? db))- (is (= "Test Database" (:name db))))))--(deftest source-resolves-table-on-demand-test- (testing "SerdesSchemaSource resolves tables by path on demand (no pre-indexing)"- (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))- table (source/resolve-table source ["Test Database" "public" "orders"])]- (is (some? table))- (is (= "orders" (:name table))))- (testing "schema-less tables"- (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))- table (source/resolve-table source ["SQLite DB" nil "orders"])]- (is (some? table))- (is (= "orders" (:name table)))))))--(deftest source-resolves-field-on-demand-test- (testing "SerdesSchemaSource resolves fields by path on demand"- (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))- field (source/resolve-field source ["Test Database" "public" "orders" "id"])]- (is (some? field))- (is (= "id" (:name field))))- (testing "schema-less fields"- (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))- field (source/resolve-field source ["SQLite DB" nil "orders" "id"])]- (is (some? field))- (is (= "id" (:name field)))))))--(deftest source-returns-nil-for-unknown-test- (testing "SerdesSchemaSource returns nil for unknown entities"- (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))]- (is (nil? (source/resolve-database source "Nonexistent")))- (is (nil? (source/resolve-table source ["DB" "x" "y"])))- (is (nil? (source/resolve-field source ["DB" "x" "y" "z"]))))))--;;; ===========================================================================-;;; fields-for-table and all-table-paths — on-demand enumeration-;;; ===========================================================================--(deftest fields-for-table-test- (testing "fields-for-table returns field paths for a table"- (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))- fields (source/fields-for-table source ["Test Database" "public" "orders"])]- (is (set? fields))- (is (contains? fields ["Test Database" "public" "orders" "id"]))- (is (contains? fields ["Test Database" "public" "orders" "total"]))))- (testing "fields-for-table works for schema-less tables"- (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))- fields (source/fields-for-table source ["SQLite DB" nil "orders"])]- (is (set? fields))- (is (contains? fields ["SQLite DB" nil "orders" "id"]))))- (testing "fields-for-table returns nil for unknown table"- (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))]- (is (nil? (source/fields-for-table source ["Nope" "x" "y"]))))))--(deftest all-table-paths-test- (testing "all-table-paths enumerates all tables across databases"- (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))- tables (source/all-table-paths source)]- (is (seq tables))- (is (some #(= ["Test Database" "public" "orders"] %) tables))- (is (some #(= ["SQLite DB" nil "orders"] %) tables)))))--(deftest all-database-names-test- (testing "all-database-names returns database names"- (let [source (serdes-schema/make-database-source (str (fixtures-path) "/databases"))]- (is (= #{"Test Database" "SQLite DB"} (set (source/all-database-names source)))))))--;;; ===========================================================================-;;; Slugified directory names — critical correctness tests-;;; ===========================================================================--(deftest slugified-db-dir-resolves-correctly-test- (testing "databases with slugified directory names resolve correctly"- (with-temp-dir- (fn [dir]- (let [db-dir (str dir "/databases")]- ;; Create a database with slugified dir but real name in YAML- (write-yaml! dir "databases" "analytics_data_warehouse" "analytics_data_warehouse.yaml"- {:name "Analytics Data Warehouse" :engine "postgres"})- ;; Create a table under it- (write-yaml! dir "databases" "analytics_data_warehouse" "schemas" "public" "tables" "orders" "orders.yaml"- {:name "orders" :schema "public"- :serdes/meta [{:id "Analytics Data Warehouse" :model "Database"}- {:id "public" :model "Schema"}- {:id "orders" :model "Table"}]})- ;; Create a field- (write-yaml! dir "databases" "analytics_data_warehouse" "schemas" "public" "tables" "orders" "fields" "id.yaml"- {:name "id" :base_type "type/Integer"- :serdes/meta [{:id "Analytics Data Warehouse" :model "Database"}- {:id "public" :model "Schema"}- {:id "orders" :model "Table"}- {:id "id" :model "Field"}]})- (let [source (serdes-schema/make-database-source db-dir)]- ;; Database resolves by real name- (is (some? (source/resolve-database source "Analytics Data Warehouse")))- ;; Table resolves using real db name- (is (some? (source/resolve-table source ["Analytics Data Warehouse" "public" "orders"])))- ;; Field resolves using real db name- (let [field (source/resolve-field source ["Analytics Data Warehouse" "public" "orders" "id"])]- (is (some? field))- (is (= "id" (:name field))))- ;; fields-for-table works- (let [fields (source/fields-for-table source ["Analytics Data Warehouse" "public" "orders"])]- (is (contains? fields ["Analytics Data Warehouse" "public" "orders" "id"])))))))))--;;; ===========================================================================-;;; Performance guards — indexing should NOT parse field/table YAMLs-;;; ===========================================================================--(deftest index-does-not-contain-tables-or-fields-test- (testing "the file index only contains databases — tables and fields are resolved on demand"- (let [db-dir (str (fixtures-path) "/databases")- {:keys [index]} (serdes-schema/build-database-dir-index db-dir)]- (is (seq (:database index)) "databases should be in the index")- (is (nil? (:table index)) "tables should NOT be in the index")- (is (nil? (:field index)) "fields should NOT be in the index"))))--;;; ===========================================================================-;;; Cache shape and lazy resolution-;;; ===========================================================================--(def ^:private not-indexed ::serdes-schema/not-indexed)--(deftest schema-model-shape-test- (testing "schema model starts with ::not-indexed for each schema, populates lazily"- (with-temp-dir- (fn [dir]- (write-yaml! dir "databases" "db" "db.yaml" {:name "DB" :engine "h2"})- (write-yaml! dir "databases" "db" "schemas" "s1" "tables" "t1" "t1.yaml"- {:name "T1" :schema "s1"})- (write-yaml! dir "databases" "db" "schemas" "s1" "tables" "t1" "fields" "f1.yaml"- {:name "F1" :base_type "type/Integer"})- (write-yaml! dir "databases" "db" "schemas" "s2" "tables" "t2" "t2.yaml"- {:name "T2" :schema "s2"})- (write-yaml! dir "databases" "db" "schemas" "s2" "tables" "t2" "fields" "f2.yaml"- {:name "F2" :base_type "type/Text"})- (let [source (serdes-schema/make-database-source (str dir "/databases"))]- ;; Initial: db-file set, all schemas ::not-indexed- (is (=? {"DB" {:db-file string?- :schemas {"s1" not-indexed- "s2" not-indexed}}}- (serdes-schema/schema-model source)))- ;; After resolving a table in s1: s1 indexed with tables, s2 untouched- (source/resolve-table source ["DB" "s1" "T1"])- (is (=? {"DB" {:schemas {"s1" {"T1" {:table-file string?- :fields not-indexed}}- "s2" not-indexed}}}- (serdes-schema/schema-model source)))- ;; After resolving fields for T1: T1 fields indexed, s2 still untouched- (source/fields-for-table source ["DB" "s1" "T1"])- (is (=? {"DB" {:schemas {"s1" {"T1" {:table-file string?- :fields {"F1" string?}}}- "s2" not-indexed}}}- (serdes-schema/schema-model source))))))))--(deftest case-insensitive-schema-resolution-test- (testing "schemas with different casing on disk vs YAML resolve correctly"- (with-temp-dir- (fn [dir]- ;; Dir name is lowercase, YAML schema is uppercase (H2 convention)- (write-yaml! dir "databases" "db" "db.yaml" {:name "DB" :engine "h2"})- (write-yaml! dir "databases" "db" "schemas" "public" "tables" "orders" "orders.yaml"- {:name "ORDERS" :schema "PUBLIC"})- (write-yaml! dir "databases" "db" "schemas" "public" "tables" "orders" "fields" "id.yaml"- {:name "ID" :base_type "type/Integer"})- (let [source (serdes-schema/make-database-source (str dir "/databases"))]- ;; Resolve by real names (uppercase) — schema re-keys from "public" to "PUBLIC"- (is (some? (source/resolve-table source ["DB" "PUBLIC" "ORDERS"])))- (is (some? (source/resolve-field source ["DB" "PUBLIC" "ORDERS" "ID"])))- (is (=? {"DB" {:schemas {"PUBLIC" {"ORDERS" {:fields {"ID" string?}}}}}}- (serdes-schema/schema-model source))))))))diff --git a/enterprise/backend/test/metabase_enterprise/checker/semantic_test.clj b/enterprise/backend/test/metabase_enterprise/checker/semantic_test.cljdeleted file mode 100644index 7f3c7775c1e..00000000000--- a/enterprise/backend/test/metabase_enterprise/checker/semantic_test.clj+++ /dev/null@@ -1,1220 +0,0 @@-(ns metabase-enterprise.checker.semantic-test- "Tests for the semantic checker.-- Uses real YAML fixture files from test_resources/yaml_checks for most tests.- Uses in-memory sources for edge cases like unresolved refs."- (:require- [clojure.java.io :as io]- [clojure.test :refer [deftest is testing]]- [metabase-enterprise.checker.format.serdes-assets :as serdes-format]- [metabase-enterprise.checker.format.serdes-schema :as serdes-schema]- [metabase-enterprise.checker.provider :as provider]- [metabase-enterprise.checker.semantic :as checker]- [metabase-enterprise.checker.source :as source]- [metabase-enterprise.checker.store :as store]- [metabase-enterprise.checker.test-helpers :as helpers]- [metabase.lib.metadata.protocols :as lib.metadata.protocols]))--(set! *warn-on-reflection* true)--;;; ===========================================================================-;;; Test Fixture Path-;;; ===========================================================================--(def ^:private test-fixtures-dir- "Path to test fixtures directory (real YAML files)."- "test_resources/yaml_checks")--(defn- fixtures-path []- (let [f (io/file test-fixtures-dir)]- (if (.isAbsolute f)- test-fixtures-dir- (.getPath (io/file (System/getProperty "user.dir") test-fixtures-dir)))))--(defn- make-test-sources- "Create both schema and assets sources from fixtures.- Returns [schema-source assets-source]."- []- [(serdes-schema/make-database-source (str (fixtures-path) "/databases"))- (serdes-format/make-source (fixtures-path))])--(defn- check-all [[schema-source assets-source]]- (checker/check-entities schema-source assets-source (serdes-format/source-index assets-source)))--(defn- check-specific [[schema-source assets-source] entity-ids]- (checker/check-entities schema-source assets-source (serdes-format/source-index assets-source) entity-ids))--;;; ===========================================================================-;;; Tests Using Real YAML Files-;;; ===========================================================================--(deftest source-index-test- (testing "Source correctly indexes databases and cards; tables/fields are on-demand"- (let [[schema-source assets-source] (make-test-sources)- index (serdes-format/source-index assets-source)]- ;; Databases are in the schema source- (is (= #{"Test Database" "SQLite DB"} (set (source/all-database-names schema-source))))- ;; Cards are in the assets index- (is (= 5 (count (:card index))) "Should have 5 cards")- ;; Tables and fields are resolved on demand via schema source- (is (= 4 (count (source/all-table-paths schema-source))) "Should have 4 tables via source")- (is (= 11 (count (source/all-field-paths schema-source))) "Should have 11 fields via source"))))--(deftest simple-mbql-query-test- (testing "Simple MBQL query on orders table validates successfully"- (let [sources (make-test-sources)- results (check-all sources)- result (get results "simple-orders")]- (is (some? result) "Card should be found")- (is (= "Simple Orders" (:name result)) "Card name should match")- (is (nil? (:error result)) (str "Should not have errors: " (:error result)))- (is (empty? (:unresolved result)) "Should not have unresolved refs")- (is (empty? (:bad-refs result)) "Should not have bad refs from lib")- ;; Check that table refs were extracted- (is (seq (get-in result [:refs :tables])) "Should have table refs"))))--(deftest native-query-test- (testing "Native SQL query validates successfully"- (let [sources (make-test-sources)- results (check-all sources)- result (get results "native-orders")]- (is (some? result) "Card should be found")- (is (= "Native Orders" (:name result)) "Card name should match")- (is (nil? (:error result)) (str "Should not have errors: " (:error result)))- (is (empty? (:unresolved result)) "Should not have unresolved refs"))))--(deftest mbql-with-joins-test- (testing "MBQL query with joins validates successfully"- (let [sources (make-test-sources)- results (check-all sources)- result (get results "orders-with-products")]- (is (some? result) "Card should be found")- (is (= "Orders With Products" (:name result)) "Card name should match")- (is (nil? (:error result)) (str "Should not have errors: " (:error result)))- (is (empty? (:unresolved result)) "Should not have unresolved refs")- ;; Check that both tables are referenced- (let [tables (get-in result [:refs :tables])]- (is (some #(re-find #"orders" %) tables) "Should reference orders table")- (is (some #(re-find #"products" %) tables) "Should reference products table")))))--(deftest all-cards-checked-test- (testing "All cards in fixtures are checked"- (let [sources (make-test-sources)- results (check-all sources)]- (is (>= (count results) 5) "Should check at least 5 cards")- (is (contains? results "simple-orders"))- (is (contains? results "native-orders"))- (is (contains? results "orders-with-products"))- (is (contains? results "schemaless-query"))- (is (contains? results "orders-with-fk")))))--;;; ===========================================================================-;;; Schema-less Database Tests-;;;-;;; Databases like SQLite don't have schemas, so tables are stored directly-;;; under databases/<db>/tables/ instead of databases/<db>/schemas/<schema>/tables/-;;; ===========================================================================--(deftest schemaless-database-index-test- (testing "Schema-less databases are indexed correctly with nil schema"- (let [[schema-source _] (make-test-sources)]- (is (= #{"Test Database" "SQLite DB"} (set (source/all-database-names schema-source))))- (is (= 4 (count (source/all-table-paths schema-source))) "Should have 4 tables via source"))))--(deftest schemaless-query-test- (testing "Query on schema-less database validates successfully"- (let [sources (make-test-sources)- results (check-all sources)- result (get results "schemaless-query")]- (is (some? result) "Card should be found")- (is (= "Schemaless Query" (:name result)) "Card name should match")- (is (nil? (:error result)) (str "Should not have errors: " (:error result)))- (is (empty? (:unresolved result)) "Should not have unresolved refs")- (is (empty? (:bad-refs result)) "Should not have bad refs"))))--;;; ===========================================================================-;;; FK Resolution Tests-;;;-;;; Foreign key references in field metadata and result_metadata must be-;;; converted from path vectors to integer IDs.-;;; ===========================================================================--(deftest fk-in-field-metadata-test- (testing "FK target in field metadata is resolved to integer ID"- (let [[schema-source assets-source] (make-test-sources)- index (serdes-format/source-index assets-source)- provider (provider/make-provider (store/make-store schema-source assets-source index))- ;; Get the product_id field which has FK to products.id- fields (lib.metadata.protocols/metadatas- provider {:lib/type :metadata/column})- fk-field (first (filter #(= "product_id" (:name %)) fields))]- (is (some? fk-field) "Should find product_id field")- (is (= :type/FK (:semantic-type fk-field)) "Should be FK type")- (is (pos-int? (:fk-target-field-id fk-field))- "FK target should be resolved to positive integer"))))--(deftest fk-in-result-metadata-test- (testing "FK target in card result_metadata is resolved to integer ID"- (let [sources (make-test-sources)- results (check-all sources)- result (get results "orders-with-fk")]- (is (some? result) "Card should be found")- (is (= "Orders With FK" (:name result)) "Card name should match")- (is (nil? (:error result)) (str "Should not have errors: " (:error result)))- (is (empty? (:unresolved result)) "Should not have unresolved refs")- (is (empty? (:bad-refs result)) "Should not have bad refs"))))--;;; ===========================================================================-;;; Tests Using In-Memory Source for Edge Cases-;;;-;;; ===========================================================================-;;; MBQL normalization tests-;;; ===========================================================================--(deftest template-tag-default-not-keywordized-test- (testing "Template tag :default values are not incorrectly keywordized as MBQL operators"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "ORDERS"] {:name "ORDERS" :schema "PUBLIC"}}- :fields {["DB" "PUBLIC" "ORDERS" "CREATED_AT"]- {:name "CREATED_AT" :base_type "type/DateTimeWithLocalTZ"- :database_type "TIMESTAMP WITH TIME ZONE"}}- :cards {"tMpLtAgDeFaUlT00card1"- {:name "Field Filter Card"- :entity_id "tMpLtAgDeFaUlT00card1"- :dataset_query- {:database "DB"- :stages [{:native "SELECT * FROM ORDERS WHERE {{date_filter}}"- :template-tags- {"date_filter"- {:type "dimension"- :dimension ["field" {} ["DB" "PUBLIC" "ORDERS" "CREATED_AT"]]- :default "past30days"- :display-name "Date Filter"- :name "date_filter"- :widget-type "date/all-options"}}- "lib/type" "mbql.stage/native"}]- "lib/type" "mbql/query"}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["tMpLtAgDeFaUlT00card1"])- result (get results "tMpLtAgDeFaUlT00card1")]- (is (nil? (:error result))- (str "Should not error: " (:error result))))))--;;; ===========================================================================-;;; Create a simple in-memory source to test unresolved references-;;; without needing temp files.-;;; ===========================================================================--(deftest unresolved-table-in-card-test- (testing "Card referencing nonexistent table is detected"- (let [entities {:databases {"Test Database" {:name "Test Database"- :engine "h2"}}- :tables {} ; No tables!- :fields {}- :cards {"test-card" {:name "Test Card"- :entity_id "test-card"- :dataset_query {:database "Test Database"- :type "query"- :query {:source-table ["Test Database" "public" "orders"]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["test-card"])- result (get results "test-card")]- (is (some? result) "Card should be found")- (is (= "Test Card" (:name result)) "Card name should match")- (is (seq (:unresolved result)) "Should have unresolved refs")- (is (some #(= :table (:type %)) (:unresolved result))- "Should have unresolved table ref"))))--(deftest unresolved-database-in-card-test- (testing "Card referencing nonexistent database is detected"- (let [entities {:databases {} ; No databases!- :tables {}- :fields {}- :cards {"test-card" {:name "Test Card"- :entity_id "test-card"- :dataset_query {:database "Nonexistent Database"- :type "query"- :query {:source-table ["Nonexistent Database" "public" "orders"]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["test-card"])- result (get results "test-card")]- (is (some? result) "Card should be found")- (is (= "Test Card" (:name result)) "Card name should match")- ;; Either has an error about missing database, or unresolved refs- (is (or (:error result)- (some #(= :database (:type %)) (:unresolved result)))- "Should have unresolved database or error"))))--(deftest check-specific-cards-test- (testing "Can check specific cards by entity-id"- (let [sources (make-test-sources)- results (check-specific sources ["simple-orders" "native-orders"])]- (is (>= (count results) 2) "Should check at least 2 cards")- (is (contains? results "simple-orders"))- (is (contains? results "native-orders"))- (is (not (contains? results "orders-with-products"))))))--;;; ===========================================================================-;;; Results Processing Tests-;;; ===========================================================================--(deftest result-status-test- (testing "Result status is computed correctly"- (is (= :ok (checker/result-status {})))- (is (= :error (checker/result-status {:error "Something went wrong"})))- (is (= :unresolved (checker/result-status {:unresolved [{:type :table}]})))- (is (= :issues (checker/result-status {:bad-refs [{:type :field}]})))))--(deftest summarize-results-test- (testing "Results are summarized correctly"- (let [results {"card1" {}- "card2" {:error "oops"}- "card3" {:unresolved [{:type :table}]}- "card4" {:bad-refs [{:type :field}]}}- summary (checker/summarize-results results)]- (is (= 4 (:total summary)))- (is (= 1 (:ok summary)))- (is (= 1 (:errors summary)))- (is (= 1 (:unresolved summary)))- (is (= 1 (:issues summary))))))--;;; ===========================================================================-;;; Format-error Tests (LLM-friendly error output)-;;; ===========================================================================--(deftest format-error-returns-nil-for-ok-test- (testing "format-error returns nil for OK results"- (is (nil? (checker/format-error ["card-1" {}])))- (is (nil? (checker/format-error ["card-1" {:refs {:tables ["t"]}}])))))--(deftest format-error-includes-card-identity-test- (testing "format-error includes card name and entity-id"- (let [output (checker/format-error ["abc123" {:name "My Card" :error "boom"}])]- (is (some? output))- (is (re-find #"My Card" output))- (is (re-find #"abc123" output)))))--(deftest format-error-includes-unresolved-refs-test- (testing "format-error includes unresolved field/table refs"- (let [output (checker/format-error- ["eid" {:name "Bad Card"- :unresolved [{:type :field :path ["db" "public" "t" "bad_col"]}- {:type :table :path ["db" "public" "missing"]}]}])]- (is (re-find #"unresolved field" output))- (is (re-find #"bad_col" output))- (is (re-find #"unresolved table" output))- (is (re-find #"missing" output)))))--(deftest format-error-includes-bad-refs-test- (testing "format-error includes bad-refs"- (let [output (checker/format-error- ["eid" {:name "Issue Card"- :bad-refs [{:type :missing-column :name "gone"}]}])]- (is (re-find #"bad ref" output))- (is (re-find #"missing-column" output)))))--(deftest format-error-includes-error-message-test- (testing "format-error includes the error message"- (let [output (checker/format-error- ["eid" {:name "Error Card" :error "Unknown database: foo"}])]- (is (re-find #"error: Unknown database: foo" output)))))--(deftest format-result-non-card-entity-test- (testing "format-result shows kind for non-card entities instead of Card ID"- (let [output (checker/format-result- ["eid" {:name "My Dashboard" :entity-id "eid" :kind :dashboard- :unresolved [{:type :collection :entity-id "bad" :message "not found"}]}])]- (is (re-find #"Kind: dashboard" output))- (is (not (re-find #"Card ID" output))))))--(deftest format-error-non-card-entity-test- (testing "format-error uses kind name for non-card entities"- (let [output (checker/format-error- ["eid" {:name "My Transform" :entity-id "eid" :kind :transform- :error "Unknown database: foo"}])]- (is (re-find #"^transform:" output))- (is (not (re-find #"^card:" output))))))--;;; ===========================================================================-;;; Sentinel ID Tests — unresolved refs get IDs so queries can be constructed-;;; ===========================================================================--(deftest sentinel-id-for-unresolved-field-test- (testing "Card with unresolved field gets sentinel ID — query builds, field flagged"- (let [entities {:databases {"Test DB" {:name "Test DB" :engine "h2"}}- :tables {["Test DB" "PUBLIC" "ORDERS"] {:name "ORDERS" :schema "PUBLIC"}}- :fields {["Test DB" "PUBLIC" "ORDERS" "ID"] {:name "ID"- :base_type "type/BigInteger"- :database_type "BIGINT"- :table_id ["Test DB" "PUBLIC" "ORDERS"]}}- ;; Card references NONEXISTENT_FIELD which doesn't exist- :cards {"bad-field" {:name "Bad Field Card"- :entity_id "bad-field"- :type "question"- :dataset_query {:database "Test DB"- :type "query"- :query {:source-table ["Test DB" "PUBLIC" "ORDERS"]- :breakout [[:field- ["Test DB" "PUBLIC" "ORDERS" "NONEXISTENT_FIELD"]- {:base-type :type/Text}]]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["bad-field"])- result (get results "bad-field")]- (is (some? result))- ;; Should NOT have an :error (query should build successfully with sentinel)- (is (nil? (:error result)) (str "Should not blow up: " (:error result)))- ;; Should have unresolved refs tracking the bad field- (is (seq (:unresolved result)) "Should track unresolved field")- (is (some #(= :field (:type %)) (:unresolved result))- "Should have unresolved field ref"))))--(deftest sentinel-id-for-unresolved-table-in-join-test- (testing "Card with unresolved table in join gets sentinel ID — query builds"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "ORDERS"] {:name "ORDERS" :schema "PUBLIC"}}- :fields {}- ;; Card joins to a nonexistent table- :cards {"bad-join" {:name "Bad Join Card"- :entity_id "bad-join"- :type "question"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "ORDERS"]- :joins [{:source-table ["DB" "PUBLIC" "NONEXISTENT"]- :alias "n"- :condition [:= 1 1]}]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["bad-join"])- result (get results "bad-join")]- (is (some? result))- ;; Should NOT have an :error (query should build with sentinel)- (is (nil? (:error result)) (str "Should not blow up: " (:error result)))- ;; Should track the unresolved table- (is (seq (:unresolved result)) "Should track unresolved table")- (is (some #(= :table (:type %)) (:unresolved result))- "Should have unresolved table ref"))))--;;; ===========================================================================-;;; Native SQL validation via deps.analysis-;;;-;;; These tests verify that deps.analysis/check-entity is properly invoked-;;; for native SQL cards (requires sql-tools.init to be loaded).-;;; ===========================================================================--(deftest native-card-bad-sql-produces-native-errors-test- (testing "Card with bad SQL produces native-errors, not validation-exception-error"- (let [entities {:databases {"Test DB" {:name "Test DB" :engine "h2"}}- :tables {["Test DB" "PUBLIC" "ORDERS"]- {:name "ORDERS" :schema "PUBLIC"}}- :fields {["Test DB" "PUBLIC" "ORDERS" "ID"]- {:name "ID" :base_type "type/BigInteger"- :database_type "BIGINT"- :table_id ["Test DB" "PUBLIC" "ORDERS"]}}- :cards {"bad-sql" {:name "Bad SQL"- :entity_id "bad-sql"- :type "question"- :dataset_query {:database "Test DB"- :type "native"- :native {:query "SELLECT * FROM ORDERS"}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["bad-sql"])- result (get results "bad-sql")]- (is (some? result))- (is (or (seq (:native-errors result))- (:error result))- "Bad SQL should produce native-errors or error, not validation-exception-error")- (is (empty? (filter #(= :validation-exception-error (:type %))- (:bad-refs result)))- "Should not have validation-exception-error in bad-refs"))))--(deftest native-card-valid-sql-no-errors-test- (testing "Card with valid native SQL produces no errors"- (let [entities {:databases {"Test DB" {:name "Test DB" :engine "h2"}}- :tables {["Test DB" "PUBLIC" "ORDERS"]- {:name "ORDERS" :schema "PUBLIC"}}- :fields {["Test DB" "PUBLIC" "ORDERS" "ID"]- {:name "ID" :base_type "type/BigInteger"- :database_type "BIGINT"- :table_id ["Test DB" "PUBLIC" "ORDERS"]}- ["Test DB" "PUBLIC" "ORDERS" "TOTAL"]- {:name "TOTAL" :base_type "type/Float"- :database_type "DOUBLE PRECISION"- :table_id ["Test DB" "PUBLIC" "ORDERS"]}}- :cards {"good-sql" {:name "Good SQL"- :entity_id "good-sql"- :type "question"- :dataset_query {:database "Test DB"- :type "native"- :native {:query "SELECT ID, TOTAL FROM ORDERS"}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["good-sql"])- result (get results "good-sql")]- (is (some? result))- (is (nil? (:error result)) (str "Should not error: " (:error result)))- (is (empty? (:native-errors result)) "Valid SQL should have no native errors")- (is (empty? (:bad-refs result)) "Valid SQL should have no bad refs"))))--;;; ===========================================================================-;;; Computed result-metadata tests-;;;-;;; When YAML has no result_metadata, the checker computes it from the query-;;; so deps.analysis can verify card subquery column references.-;;; ===========================================================================--(deftest mbql-card-computes-result-metadata-test- (testing "MBQL card without result_metadata computes columns from query"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "ORDERS"] {:name "ORDERS" :schema "PUBLIC"}}- :fields {["DB" "PUBLIC" "ORDERS" "ID"]- {:name "ID" :base_type "type/BigInteger" :database_type "BIGINT"- :table_id ["DB" "PUBLIC" "ORDERS"]}- ["DB" "PUBLIC" "ORDERS" "TOTAL"]- {:name "TOTAL" :base_type "type/Float" :database_type "DOUBLE PRECISION"- :table_id ["DB" "PUBLIC" "ORDERS"]}}- ;; Card with MBQL query but no result_metadata- :cards {"mbql-card" {:name "MBQL Card"- :entity_id "mbql-card"- :type "question"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "ORDERS"]}}}- ;; Native card referencing the MBQL card's columns- "native-card" {:name "Native Ref"- :entity_id "native-card"- :type "question"- :dataset_query {:database "DB"- :type "native"- :native {:query "SELECT ID FROM {{#1-mbql}}"- :template-tags {"#1-mbql" {:type :card- :name "#1-mbql"- :card-id "mbql-card"}}}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["native-card"])- result (get results "native-card")]- (is (some? result))- ;; ID exists in ORDERS and in the MBQL card's computed result-metadata,- ;; so it should NOT be flagged as missing- (is (not (some #(and (= :missing-column (:type %))- (= "ID" (:name %)))- (:native-errors result)))- "ID should not be flagged — it exists in the MBQL card's computed columns"))))--(deftest mbql-card-named-aggregation-in-result-metadata-test- (testing "Named aggregation columns appear in computed result-metadata"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "ORDERS"] {:name "ORDERS" :schema "PUBLIC"}}- :fields {["DB" "PUBLIC" "ORDERS" "TOTAL"]- {:name "TOTAL" :base_type "type/Float" :database_type "DOUBLE PRECISION"- :table_id ["DB" "PUBLIC" "ORDERS"]}}- ;; Card with named aggregation but no result_metadata- :cards {"agg-card" {:name "Agg Card"- :entity_id "agg-card"- :type "question"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "ORDERS"]- :aggregation [[:sum {:name "total_revenue"- :display-name "Total Revenue"}- [:field- ["DB" "PUBLIC" "ORDERS" "TOTAL"]- {:base-type :type/Float}]]]}}}- ;; Native card referencing the named aggregation column- "ref-card" {:name "Ref Card"- :entity_id "ref-card"- :type "question"- :dataset_query {:database "DB"- :type "native"- :native {:query "SELECT total_revenue FROM {{#1-agg}}"- :template-tags {"#1-agg" {:type :card- :name "#1-agg"- :card-id "agg-card"}}}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["ref-card"])- result (get results "ref-card")]- (is (some? result))- (is (not (some #(and (= :missing-column (:type %))- (= "total_revenue" (:name %)))- (:native-errors result)))- "total_revenue should not be flagged — it's a named aggregation in the source card"))))--(deftest native-card-without-result-metadata-no-false-positives-test- (testing "Native card without result_metadata doesn't produce false positive errors"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "ORDERS"] {:name "ORDERS" :schema "PUBLIC"}}- :fields {["DB" "PUBLIC" "ORDERS" "ID"]- {:name "ID" :base_type "type/BigInteger" :database_type "BIGINT"- :table_id ["DB" "PUBLIC" "ORDERS"]}- ["DB" "PUBLIC" "ORDERS" "TOTAL"]- {:name "TOTAL" :base_type "type/Float" :database_type "DOUBLE PRECISION"- :table_id ["DB" "PUBLIC" "ORDERS"]}}- :cards {"sql-card" {:name "SQL Card"- :entity_id "sql-card"- :type "question"- :dataset_query {:database "DB"- :type "native"- :native {:query "SELECT ID, TOTAL FROM ORDERS"}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["sql-card"])- result (get results "sql-card")]- (is (some? result))- (is (nil? (:error result)))- (is (empty? (:bad-refs result)) "Valid native SQL should have no bad refs"))))--;;; ===========================================================================-;;; Measure and Segment reference tests-;;; ===========================================================================--(deftest measure-ref-resolved-test- (testing "Card referencing a known measure resolves without errors"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "T"] {:name "T" :schema "PUBLIC"}}- :fields {}- :cards {"card-1" {:name "Uses Measure"- :entity_id "card-1"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]- :aggregation [["measure" "mSrAvgProdPrice00008x"]]}}}}- :measures {"mSrAvgProdPrice00008x" {:name "Avg Product Price"- :entity_id "mSrAvgProdPrice00008x"}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["card-1"])- result (get results "card-1")]- (is (nil? (:error result)) (str "Should not error: " (:error result)))- (is (empty? (filter #(= :measure (:type %)) (or (:unresolved result) [])))- "Should not have unresolved measure refs"))))--(deftest measure-ref-missing-test- (testing "Card referencing unknown measure is flagged"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "T"] {:name "T" :schema "PUBLIC"}}- :fields {}- :cards {"card-1" {:name "Bad Measure Ref"- :entity_id "card-1"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]- :aggregation [["measure" "xXxNoExIsT0MeAsUrExx1"]]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["card-1"])- result (get results "card-1")]- (is (some #(= :measure (:type %)) (or (:unresolved result) []))- "Should have unresolved measure ref"))))--(deftest segment-ref-resolved-test- (testing "Card referencing a known segment resolves without errors"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "T"] {:name "T" :schema "PUBLIC"}}- :fields {}- :cards {"card-1" {:name "Uses Segment"- :entity_id "card-1"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]- :filter ["segment" "aB3kLmN9pQrStUvWxYz1a"]}}}}- :segments {"aB3kLmN9pQrStUvWxYz1a" {:name "Known Segment"- :entity_id "aB3kLmN9pQrStUvWxYz1a"}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["card-1"])- result (get results "card-1")]- (is (nil? (:error result)) (str "Should not error: " (:error result)))- (is (empty? (filter #(= :segment (:type %)) (or (:unresolved result) [])))- "Should not have unresolved segment refs"))))--(deftest segment-ref-missing-test- (testing "Card referencing unknown segment is flagged"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "T"] {:name "T" :schema "PUBLIC"}}- :fields {}- :cards {"card-1" {:name "Bad Segment Ref"- :entity_id "card-1"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]- :filter ["segment" "xXxNoExIsT0SeGmEnTx12"]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["card-1"])- result (get results "card-1")]- (is (some #(= :segment (:type %)) (or (:unresolved result) []))- "Should have unresolved segment ref"))))--;;; ===========================================================================-;;; Transitive entity ref tests-;;; ===========================================================================--(deftest transitive-snippet-refs-test- (testing "Card sourcing from another card that uses a snippet surfaces that snippet transitively"- ;; Entity-ids must be 21-char NanoIDs to pass portable-id? checks- (let [inner-eid "tRaNsInNeRsNiPpEt0001"- outer-eid "tRaNsOuTeRsNiPpEt0001"- snip-eid "tRaNsSnIpPeTiD0000001"- entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "ORDERS"] {:name "ORDERS" :schema "PUBLIC"}}- :fields {["DB" "PUBLIC" "ORDERS" "ID"] {:name "ID" :base_type "type/BigInteger"- :database_type "BIGINT"}}- :cards {inner-eid {:name "Inner Card"- :entity_id inner-eid- :dataset_query {:database "DB"- :stages [{:native "SELECT * FROM ORDERS WHERE {{snippet: My Filter}}"- :template-tags {"snippet: My Filter"- {:type "snippet"- :snippet-name "My Filter"- :snippet-id snip-eid}}- "lib/type" "mbql.stage/native"}]- "lib/type" "mbql/query"}- :result_metadata [{:name "ID" :base_type "type/BigInteger"}]}- outer-eid {:name "Outer Card"- :entity_id outer-eid- :dataset_query {:database "DB"- :stages [{:source-card inner-eid- "lib/type" "mbql.stage/mbql"}]- "lib/type" "mbql/query"}}}- :snippets {snip-eid {:name "My Filter" :content "1=1"}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index [outer-eid inner-eid])- outer (get results outer-eid)]- (is (= ["Inner Card"] (get-in outer [:refs :source-cards]))- "Should list inner card as source")- (is (= ["My Filter"] (get-in outer [:refs :snippets]))- "Should transitively surface the inner card's snippet"))))--(deftest transitive-measure-refs-test- (testing "Card sourcing from another card that uses measures/segments surfaces them transitively"- (let [inner-eid "tRaNsInNeRmEaSuRe0001"- outer-eid "tRaNsOuTeRmEaSuRe0001"- msr-eid "tRaNsMeAsUrEiD0000001"- seg-eid "tRaNsSeGmEnTiD0000001"- entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "T"] {:name "T" :schema "PUBLIC"}}- :fields {}- :cards {inner-eid {:name "Inner Metrics"- :entity_id inner-eid- :dataset_query {:database "DB"- :stages [{:source-table ["DB" "PUBLIC" "T"]- :aggregation [["measure" msr-eid]]- :filters [["segment" seg-eid]]- "lib/type" "mbql.stage/mbql"}]- "lib/type" "mbql/query"}- :result_metadata [{:name "count" :base_type "type/Integer"}]}- outer-eid {:name "Outer Query"- :entity_id outer-eid- :dataset_query {:database "DB"- :stages [{:source-card inner-eid- "lib/type" "mbql.stage/mbql"}]- "lib/type" "mbql/query"}}}}- [schema assets index] (helpers/make-sources-and-index entities)- index (assoc index- :measure {msr-eid :memory}- :segment {seg-eid :memory})- results (checker/check-entities schema assets index [outer-eid inner-eid])- outer (get results outer-eid)]- (is (= ["Inner Metrics"] (get-in outer [:refs :source-cards])))- (is (= [msr-eid] (get-in outer [:refs :measures]))- "Should transitively surface the inner card's measure")- (is (= [seg-eid] (get-in outer [:refs :segments]))- "Should transitively surface the inner card's segment"))))--;;; ===========================================================================-;;; Collection reference tests-;;; ===========================================================================--(deftest valid-collection-id-test- (testing "Card with valid collection_id passes"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "T"] {:name "T" :schema "PUBLIC"}}- :fields {}- :cards {"card-1" {:name "Good Card"- :entity_id "card-1"- :collection_id "coll-1"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- ;; Index includes the collection- index (assoc index :collection {"coll-1" :memory})- results (checker/check-entities schema assets index ["card-1"])- result (get results "card-1")]- (is (nil? (:error result)))- (is (empty? (:unresolved result)) "Valid collection_id should not produce unresolved refs"))))--(deftest missing-collection-id-test- (testing "Card with collection_id pointing to nonexistent entity is flagged"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "T"] {:name "T" :schema "PUBLIC"}}- :fields {}- :cards {"card-1" {:name "Bad Collection Card"- :entity_id "card-1"- :collection_id "nonexistent"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["card-1"])- result (get results "card-1")]- (is (seq (:unresolved result)) "Missing collection should be flagged")- (is (some #(= :collection (:type %)) (:unresolved result))- "Should have unresolved collection ref"))))--(deftest collection-id-points-to-card-test- (testing "Card with collection_id pointing to another card (not a collection) is flagged"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "T"] {:name "T" :schema "PUBLIC"}}- :fields {}- :cards {"card-1" {:name "Card With Bad Collection"- :entity_id "card-1"- :collection_id "card-2"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]}}}- "card-2" {:name "Other Card"- :entity_id "card-2"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["card-1"])- result (get results "card-1")]- (is (seq (:unresolved result)) "collection_id pointing to a card should be flagged")- (is (some #(and (= :collection (:type %))- (re-find #"card" (or (:message %) "")))- (:unresolved result))- "Error should mention it points to a card"))))--(deftest nil-collection-id-test- (testing "Card with nil collection_id is fine"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "T"] {:name "T" :schema "PUBLIC"}}- :fields {}- :cards {"card-1" {:name "Root Card"- :entity_id "card-1"- :collection_id nil- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["card-1"])- result (get results "card-1")]- (is (nil? (:error result)))- (is (empty? (:unresolved result)) "nil collection_id should be fine"))))--;;; ===========================================================================-;;; Dashboard ID reference tests-;;; ===========================================================================--(deftest card-with-missing-dashboard-id-test- (testing "Card with dashboard_id pointing to nonexistent dashboard is flagged"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "T"] {:name "T" :schema "PUBLIC"}}- :fields {}- :cards {"card-1" {:name "Orphaned Card"- :entity_id "card-1"- :dashboard_id "nonexistent-dash"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["card-1"])- result (get results "card-1")]- (is (seq (:unresolved result)) "Missing dashboard_id should be flagged")- (is (some #(= :dashboard (:type %)) (:unresolved result))- "Should have unresolved dashboard ref"))))--(deftest card-with-valid-dashboard-id-test- (testing "Card with dashboard_id pointing to a known dashboard passes"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "T"] {:name "T" :schema "PUBLIC"}}- :fields {}- :cards {"card-1" {:name "Dashboard Card"- :entity_id "card-1"- :dashboard_id "dash-1"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- index (assoc index :dashboard {"dash-1" :memory})- results (checker/check-entities schema assets index ["card-1"])- result (get results "card-1")]- (is (nil? (:error result)))- (is (empty? (:unresolved result)) "Valid dashboard_id should not produce unresolved refs"))))--(deftest card-with-null-dashboard-id-test- (testing "Card with nil dashboard_id is fine"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "T"] {:name "T" :schema "PUBLIC"}}- :fields {}- :cards {"card-1" {:name "Standalone Card"- :entity_id "card-1"- :dashboard_id nil- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["card-1"])- result (get results "card-1")]- (is (nil? (:error result)))- (is (empty? (:unresolved result)) "nil dashboard_id should be fine"))))--(deftest card-with-dashboard-id-pointing-to-card-test- (testing "Card with dashboard_id pointing to another card (not a dashboard) is flagged"- (let [entities {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "T"] {:name "T" :schema "PUBLIC"}}- :fields {}- :cards {"card-1" {:name "Bad Dash Ref"- :entity_id "card-1"- :dashboard_id "card-2"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]}}}- "card-2" {:name "Other Card"- :entity_id "card-2"- :dataset_query {:database "DB"- :type "query"- :query {:source-table ["DB" "PUBLIC" "T"]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["card-1"])- result (get results "card-1")]- (is (seq (:unresolved result)) "dashboard_id pointing to a card should be flagged")- (is (some #(and (= :dashboard (:type %))- (re-find #"card" (or (:message %) "")))- (:unresolved result))- "Error should mention it points to a card"))))--(deftest dashboard-bad-collection-id-test- (testing "Dashboard with invalid collection_id is flagged"- (let [entities {:databases {} :tables {} :fields {} :cards {}- :dashboards {"dash-1" {:name "My Dashboard"- :entity_id "dash-1"- :collection_id "nonexistent"}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index)]- (is (contains? results "dash-1") "Dashboard should be in results")- (let [result (get results "dash-1")]- (is (seq (:unresolved result)) "Dashboard with bad collection_id should be flagged")- (is (some #(= :collection (:type %)) (:unresolved result)))))))--(deftest dashboard-valid-collection-id-not-flagged-test- (testing "Dashboard with valid collection_id passes"- (let [entities {:databases {} :tables {} :fields {} :cards {}- :dashboards {"dash-1" {:name "Good Dashboard"- :entity_id "dash-1"- :collection_id "coll-1"}}- :collections {"coll-1" {:name "My Collection"- :entity_id "coll-1"}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index)]- (is (= :ok (checker/result-status (get results "dash-1")))- "Dashboard with valid collection_id should be OK"))))--;;; ===========================================================================-;;; Dashboard semantic validation tests-;;; ===========================================================================--(defn- with-temp-dashboard- "Run checks with a dashboard entity provided via in-memory source.- `dashboard-data` is a map representing the dashboard.- `extra-index` is merged into the index (e.g. to add cards).- Calls `(f results)` with the check results."- [dashboard-data extra-index f]- (let [entity-id (:entity_id dashboard-data)- entities {:databases {}- :tables {}- :fields {}- :cards {}- :dashboards {entity-id dashboard-data}}- [schema assets base-index] (helpers/make-sources-and-index entities)- index (merge base-index- {:collection {}- :card {}}- extra-index)- results (checker/check-entities schema assets index)]- (f results)))--(deftest dashboard-card-ref-valid-test- (testing "Dashboard with valid card_id refs passes"- (with-temp-dashboard- {:name "Good Dashboard"- :entity_id "dash-1"- :tabs [{:entity_id "tab-1" :name "Tab 1" :position 0}]- :dashcards [{:entity_id "dc-1"- :card_id "card-1"- :dashboard_tab_id ["dash-1" "tab-1"]- :row 0 :col 0 :size_x 12 :size_y 6}]}- {:card {"card-1" :memory}}- (fn [results]- (is (= :ok (checker/result-status (get results "dash-1")))- "Dashboard with valid refs should be OK")))))--(deftest dashboard-card-ref-missing-test- (testing "Dashboard with card_id pointing to unknown card is flagged"- (with-temp-dashboard- {:name "Bad Card Ref"- :entity_id "dash-1"- :tabs [{:entity_id "tab-1" :name "Tab 1" :position 0}]- :dashcards [{:entity_id "dc-1"- :card_id "nonexistent-card"- :dashboard_tab_id ["dash-1" "tab-1"]- :row 0 :col 0 :size_x 12 :size_y 6}]}- {}- (fn [results]- (is (contains? results "dash-1"))- (let [result (get results "dash-1")]- (is (some #(= :dashcard-card-ref (:type %)) (:unresolved result))- "Should flag missing card ref"))))))--(deftest dashboard-virtual-card-null-card-id-test- (testing "Dashboard with null card_id (heading/text) is fine"- (with-temp-dashboard- {:name "Headings Dashboard"- :entity_id "dash-1"- :tabs [{:entity_id "tab-1" :name "Tab 1" :position 0}]- :dashcards [{:entity_id "dc-1"- :card_id nil- :dashboard_tab_id ["dash-1" "tab-1"]- :row 0 :col 0 :size_x 24 :size_y 1}]}- {}- (fn [results]- (is (= :ok (checker/result-status (get results "dash-1")))- "Virtual card with null card_id should be fine")))))--(deftest dashboard-tab-ref-invalid-test- (testing "Dashboard with dashcard referencing nonexistent tab is flagged"- (with-temp-dashboard- {:name "Bad Tab Ref"- :entity_id "dash-1"- :tabs [{:entity_id "tab-1" :name "Tab 1" :position 0}]- :dashcards [{:entity_id "dc-1"- :card_id nil- :dashboard_tab_id ["dash-1" "wrong-tab"]- :row 0 :col 0 :size_x 12 :size_y 6}]}- {}- (fn [results]- (is (contains? results "dash-1"))- (let [result (get results "dash-1")]- (is (some #(= :dashcard-tab-ref (:type %)) (:unresolved result))- "Should flag bad tab ref"))))))--(deftest dashboard-grid-overflow-test- (testing "Dashboard with dashcard extending beyond grid width is flagged"- (with-temp-dashboard- {:name "Grid Overflow"- :entity_id "dash-1"- :tabs [{:entity_id "tab-1" :name "Tab 1" :position 0}]- :dashcards [{:entity_id "dc-1"- :card_id nil- :dashboard_tab_id ["dash-1" "tab-1"]- :row 0 :col 20 :size_x 12 :size_y 6}]}- {}- (fn [results]- (is (contains? results "dash-1"))- (let [result (get results "dash-1")]- (is (some #(= :dashcard-grid (:type %)) (:unresolved result))- "Should flag grid overflow"))))))--(deftest dashboard-grid-negative-col-test- (testing "Dashboard with negative col is flagged"- (with-temp-dashboard- {:name "Negative Col"- :entity_id "dash-1"- :tabs []- :dashcards [{:entity_id "dc-1"- :card_id nil- :row 0 :col -1 :size_x 12 :size_y 6}]}- {}- (fn [results]- (is (contains? results "dash-1"))- (let [result (get results "dash-1")]- (is (some #(= :dashcard-grid (:type %)) (:unresolved result))- "Should flag negative col"))))))--(deftest dashboard-valid-full-width-test- (testing "Dashboard with full-width card (col=0, size_x=24) passes"- (with-temp-dashboard- {:name "Full Width"- :entity_id "dash-1"- :tabs [{:entity_id "tab-1" :name "Tab 1" :position 0}]- :dashcards [{:entity_id "dc-1"- :card_id nil- :dashboard_tab_id ["dash-1" "tab-1"]- :row 0 :col 0 :size_x 24 :size_y 1}]}- {}- (fn [results]- (is (= :ok (checker/result-status (get results "dash-1")))- "Full-width card should be valid")))))--;;; ===========================================================================-;;; Transform query validation tests-;;; ===========================================================================--(defn- with-temp-transform- "Run checks with a transform entity provided via in-memory source.- `transform-data` is a map representing the transform.- `extra-index` is merged into the index (e.g. to add databases).- Calls `(f results)` with the check results."- [transform-data extra-index f]- (let [entity-id (:entity_id transform-data)- entities {:databases (get extra-index :_databases {})- :tables (get extra-index :_tables {})- :fields (get extra-index :_fields {})- :cards {}- :transforms {entity-id transform-data}}- [schema assets base-index] (helpers/make-sources-and-index entities)- index (merge base-index- {:collection {}- :card {}}- (dissoc extra-index :_databases :_tables :_fields))- results (checker/check-entities schema assets index)]- (f results)))--(deftest transform-bad-native-sql-test- (testing "Transform with invalid native SQL is flagged"- (with-temp-transform- {:name "Bad SQL Transform"- :entity_id "tx-1"- :source_database_id "Test DB"- :source {:type "query"- :query {"lib/type" "mbql/query"- :database "Test DB"- :stages [{"lib/type" "mbql.stage/native"- :native "SELET * FROM ORDERS"}]}}- :target {:type "table" :schema "TRANSFORMS" :name "bad_transform"}- :serdes/meta [{:id "tx-1" :model "Transform"}]}- {:_databases {"Test DB" {:name "Test DB" :engine "h2"}}- :_tables {["Test DB" "PUBLIC" "ORDERS"] {:name "ORDERS" :schema "PUBLIC"}}- :_fields {}- :database {"Test DB" :memory}}- (fn [results]- (is (contains? results "tx-1") "Transform with bad SQL should be in results")- (let [result (get results "tx-1")]- (is (or (seq (:native-errors result))- (:error result)- (seq (:unresolved result)))- "Should have native errors, error, or unresolved refs"))))))--(deftest transform-valid-native-sql-test- (testing "Transform with valid native SQL passes"- (with-temp-transform- {:name "Good SQL Transform"- :entity_id "tx-1"- :source_database_id "Test DB"- :source {:type "query"- :query {"lib/type" "mbql/query"- :database "Test DB"- :stages [{"lib/type" "mbql.stage/native"- :native "SELECT ID, TOTAL FROM ORDERS"}]}}- :target {:type "table" :schema "TRANSFORMS" :name "good_transform"}- :serdes/meta [{:id "tx-1" :model "Transform"}]}- {:_databases {"Test DB" {:name "Test DB" :engine "h2"}}- :_tables {["Test DB" "PUBLIC" "ORDERS"] {:name "ORDERS" :schema "PUBLIC"}}- :_fields {["Test DB" "PUBLIC" "ORDERS" "ID"]- {:name "ID" :base_type "type/BigInteger" :database_type "BIGINT"- :table_id ["Test DB" "PUBLIC" "ORDERS"]}- ["Test DB" "PUBLIC" "ORDERS" "TOTAL"]- {:name "TOTAL" :base_type "type/Float" :database_type "DOUBLE PRECISION"- :table_id ["Test DB" "PUBLIC" "ORDERS"]}}- :database {"Test DB" :memory}}- (fn [results]- (is (= :ok (checker/result-status (get results "tx-1")))- "Transform with valid SQL should be OK")))))--(deftest transform-mbql-unresolved-table-test- (testing "Transform with MBQL referencing nonexistent table is flagged"- (with-temp-transform- {:name "Bad MBQL Transform"- :entity_id "tx-1"- :source_database_id "Test DB"- :source {:type "query"- :query {"lib/type" "mbql/query"- :database "Test DB"- :stages [{"lib/type" "mbql.stage/mbql"- :source-table ["Test DB" "PUBLIC" "NONEXISTENT"]}]}}- :target {:type "table" :schema "TRANSFORMS" :name "bad_mbql_transform"}- :serdes/meta [{:id "tx-1" :model "Transform"}]}- {:_databases {"Test DB" {:name "Test DB" :engine "h2"}}- :_tables {}- :_fields {}- :database {"Test DB" :memory}}- (fn [results]- (is (contains? results "tx-1") "Transform with unresolved table should be in results")- (let [result (get results "tx-1")]- (is (seq (:unresolved result)) "Should have unresolved refs"))))))--(deftest transform-missing-database-test- (testing "Transform referencing nonexistent database is flagged"- (with-temp-transform- {:name "Bad DB Transform"- :entity_id "tx-1"- :source_database_id "Nonexistent DB"- :source {:type "query"- :query {"lib/type" "mbql/query"- :database "Nonexistent DB"- :stages [{"lib/type" "mbql.stage/native"- :native "SELECT 1"}]}}- :target {:type "table" :schema "TRANSFORMS" :name "bad_db"}- :serdes/meta [{:id "tx-1" :model "Transform"}]}- {}- (fn [results]- (is (contains? results "tx-1") "Transform with missing DB should be in results")- (let [result (get results "tx-1")]- (is (some #(= :database (:type %)) (:unresolved result))- "Should flag missing database"))))))--;;; ===========================================================================-;;; Multi-database tests-;;;-;;; The provider's `(database)` method must return the correct database for the-;;; entity being checked, not just the first database in the store.-;;; ===========================================================================--(deftest multi-database-cards-test- (testing "Cards from different databases each get the correct database metadata"- (let [entities {:databases {"H2 DB" {:name "H2 DB" :engine "h2"}- "Postgres DB" {:name "Postgres DB" :engine "postgres"}}- :tables {["H2 DB" "PUBLIC" "ORDERS"]- {:name "ORDERS" :schema "PUBLIC"}- ["Postgres DB" "public" "customers"]- {:name "customers" :schema "public"}}- :fields {["H2 DB" "PUBLIC" "ORDERS" "ID"]- {:name "ID" :base_type "type/BigInteger"- :database_type "BIGINT"- :table_id ["H2 DB" "PUBLIC" "ORDERS"]}- ["Postgres DB" "public" "customers" "id"]- {:name "id" :base_type "type/BigInteger"- :database_type "bigint"- :table_id ["Postgres DB" "public" "customers"]}}- :cards {"h2-card" {:name "H2 Card"- :entity_id "h2-card"- :type "question"- :dataset_query {:database "H2 DB"- :type "query"- :query {:source-table ["H2 DB" "PUBLIC" "ORDERS"]}}}- "postgres-card" {:name "Postgres Card"- :entity_id "postgres-card"- :type "question"- :dataset_query {:database "Postgres DB"- :type "query"- :query {:source-table ["Postgres DB" "public" "customers"]}}}}}- [schema assets index] (helpers/make-sources-and-index entities)- results (checker/check-entities schema assets index ["h2-card" "postgres-card"])- h2-result (get results "h2-card")- pg-result (get results "postgres-card")]- (is (some? h2-result) "H2 card should be checked")- (is (some? pg-result) "Postgres card should be checked")- (is (nil? (:error h2-result)) (str "H2 card should not error: " (:error h2-result)))- (is (nil? (:error pg-result)) (str "Postgres card should not error: " (:error pg-result)))- (is (empty? (:bad-refs h2-result)) "H2 card should have no bad refs")- (is (empty? (:bad-refs pg-result)) "Postgres card should have no bad refs"))))--;;; ===========================================================================-;;; REPL Helpers-;;; ===========================================================================--(comment- ;; Run all tests- (clojure.test/run-tests 'metabase-enterprise.checker.semantic-test)-- ;; Check fixtures path- (fixtures-path)-- ;; Manually check all cards in fixtures- (def sources (make-test-sources))- (check-all sources))diff --git a/enterprise/backend/test/metabase_enterprise/checker/store_test.clj b/enterprise/backend/test/metabase_enterprise/checker/store_test.cljdeleted file mode 100644index 708cdef8ab2..00000000000--- a/enterprise/backend/test/metabase_enterprise/checker/store_test.clj+++ /dev/null@@ -1,211 +0,0 @@-(ns metabase-enterprise.checker.store-test- "Tests for the checker store — ID registry, index queries, entity loading/caching."- (:require- [clojure.test :refer [deftest is testing]]- [metabase-enterprise.checker.store :as store]- [metabase-enterprise.checker.test-helpers :as helpers]))--(set! *warn-on-reflection* true)--;;; ===========================================================================-;;; Test data-;;; ===========================================================================--(def ^:private test-entities- {:databases {"DB" {:name "DB" :engine "h2"}}- :tables {["DB" "PUBLIC" "ORDERS"] {:name "ORDERS" :schema "PUBLIC"}- ["DB" "PUBLIC" "PRODUCTS"] {:name "PRODUCTS" :schema "PUBLIC"}}- :fields {["DB" "PUBLIC" "ORDERS" "ID"] {:name "ID" :base_type "type/BigInteger"}- ["DB" "PUBLIC" "ORDERS" "TOTAL"] {:name "TOTAL" :base_type "type/Float"}- ["DB" "PUBLIC" "PRODUCTS" "ID"] {:name "ID" :base_type "type/BigInteger"}- ["DB" "PUBLIC" "PRODUCTS" "CATEGORY"] {:name "CATEGORY" :base_type "type/Text"}}- :cards {"card-1" {:name "Card 1" :entity_id "card-1"}}})--(defn- make-test-store []- (let [[schema assets index] (helpers/make-sources-and-index test-entities)]- (store/make-store schema assets index)))--;;; ===========================================================================-;;; Index query tests-;;; ===========================================================================--(deftest all-refs-test- (let [store (make-test-store)]- (testing "all-refs returns refs for each kind"- (is (= ["DB"] (store/all-database-names store)))- (is (= #{"card-1"} (set (store/all-card-ids store))))- (is (= 2 (count (store/all-table-paths store))))- (is (= 4 (count (store/all-field-paths store)))))))--(deftest exists?-test- (let [store (make-test-store)]- (testing "known refs return true"- (is (store/exists? store :database "DB"))- (is (store/exists? store :table ["DB" "PUBLIC" "ORDERS"]))- (is (store/exists? store :card "card-1")))- (testing "unknown refs return false"- (is (not (store/exists? store :database "Nonexistent")))- (is (not (store/exists? store :table ["DB" "PUBLIC" "MISSING"])))- (is (not (store/exists? store :card "no-such-card"))))))--(deftest index-kind-of-test- (let [store (make-test-store)]- (testing "returns the kind for known entity-ids"- (is (= :card (store/index-kind-of store "card-1"))))- (testing "returns nil for unknown entity-ids"- (is (nil? (store/index-kind-of store "unknown"))))))--(deftest index-kind-of-multi-kind-test- (testing "index-kind-of distinguishes between kinds"- (let [empty-entities {:databases {} :tables {} :fields {} :cards {}}- [schema assets _] (helpers/make-sources-and-index empty-entities)- index {:card {"eid-1" :memory}- :dashboard {"eid-2" :memory}- :collection {"eid-3" :memory}- :document {"eid-4" :memory}}- store (store/make-store schema assets index)]- (is (= :card (store/index-kind-of store "eid-1")))- (is (= :dashboard (store/index-kind-of store "eid-2")))- (is (= :collection (store/index-kind-of store "eid-3")))- (is (= :document (store/index-kind-of store "eid-4"))))))--(deftest index-file-test- (let [store (make-test-store)]- (testing "returns the file path for known refs"- (is (= :memory (store/index-file store :database "DB")))- (is (= :memory (store/index-file store :card "card-1"))))- (testing "returns nil for unknown refs"- (is (nil? (store/index-file store :database "Nonexistent"))))))--(deftest fields-for-table-test- (let [store (make-test-store)]- (testing "returns field paths for a known table"- (is (= #{["DB" "PUBLIC" "ORDERS" "ID"]- ["DB" "PUBLIC" "ORDERS" "TOTAL"]}- (store/fields-for-table store ["DB" "PUBLIC" "ORDERS"]))))- (testing "returns field paths for another table"- (is (= #{["DB" "PUBLIC" "PRODUCTS" "ID"]- ["DB" "PUBLIC" "PRODUCTS" "CATEGORY"]}- (store/fields-for-table store ["DB" "PUBLIC" "PRODUCTS"]))))- (testing "returns nil for unknown table"- (is (nil? (store/fields-for-table store ["DB" "PUBLIC" "MISSING"]))))))--;;; ===========================================================================-;;; ID registry tests-;;; ===========================================================================--(deftest get-or-assign-returns-stable-ids-test- (let [store (make-test-store)]- (testing "first call assigns an ID"- (let [id (store/get-or-assign! store :database "DB")]- (is (pos-int? id))))- (testing "second call returns the same ID"- (let [id1 (store/get-or-assign! store :database "DB")- id2 (store/get-or-assign! store :database "DB")]- (is (= id1 id2))))))--(deftest different-refs-get-different-ids-test- (let [store (make-test-store)]- (testing "different refs get different IDs"- (let [id1 (store/get-or-assign! store :table ["DB" "PUBLIC" "ORDERS"])- id2 (store/get-or-assign! store :table ["DB" "PUBLIC" "PRODUCTS"])]- (is (not= id1 id2))))))--(deftest id-ref-roundtrip-test- (let [store (make-test-store)]- (testing "id->ref and ref->id are inverses"- (let [ref ["DB" "PUBLIC" "ORDERS"]- id (store/get-or-assign! store :table ref)]- (is (= ref (store/id->ref store :table id)))- (is (= id (store/ref->id store :table ref)))))))--(deftest ref->id-returns-nil-before-assignment-test- (let [store (make-test-store)]- (testing "ref->id returns nil for unassigned refs"- (is (nil? (store/ref->id store :table ["DB" "PUBLIC" "ORDERS"]))))- (testing "id->ref returns nil for unassigned IDs"- (is (nil? (store/id->ref store :table 999))))))--(deftest ids-are-unique-across-kinds-test- (let [store (make-test-store)]- (testing "IDs don't collide across different kinds"- (let [db-id (store/get-or-assign! store :database "DB")- table-id (store/get-or-assign! store :table ["DB" "PUBLIC" "ORDERS"])- card-id (store/get-or-assign! store :card "card-1")]- (is (= 3 (count (distinct [db-id table-id card-id]))))))))--;;; ===========================================================================-;;; Entity loading and caching tests-;;; ===========================================================================--(deftest load-database-test- (let [store (make-test-store)]- (testing "load-database! returns data with assigned :id"- (let [data (store/load-database! store "DB")]- (is (some? data))- (is (= "DB" (:name data)))- (is (= "h2" (:engine data)))- (is (pos-int? (:id data)))))- (testing "load-database! returns nil for unknown database"- (is (nil? (store/load-database! store "Nonexistent"))))))--(deftest load-table-test- (let [store (make-test-store)]- (testing "load-table! returns data with :id and :db_id"- (let [data (store/load-table! store ["DB" "PUBLIC" "ORDERS"])]- (is (some? data))- (is (= "ORDERS" (:name data)))- (is (pos-int? (:id data)))- (is (pos-int? (:db_id data)))))- (testing "load-table! returns nil for unknown table"- (is (nil? (store/load-table! store ["DB" "PUBLIC" "MISSING"]))))))--(deftest load-field-test- (let [store (make-test-store)]- (testing "load-field! returns data with :id and :table_id"- (let [data (store/load-field! store ["DB" "PUBLIC" "ORDERS" "ID"])]- (is (some? data))- (is (= "ID" (:name data)))- (is (pos-int? (:id data)))- (is (pos-int? (:table_id data)))))- (testing "load-field! returns nil for unknown field"- (is (nil? (store/load-field! store ["DB" "PUBLIC" "ORDERS" "MISSING"]))))))--(deftest load-card-test- (let [store (make-test-store)]- (testing "load-card! returns data with assigned :id"- (let [data (store/load-card! store "card-1")]- (is (some? data))- (is (= "Card 1" (:name data)))- (is (pos-int? (:id data)))))- (testing "load-card! returns nil for unknown card"- (is (nil? (store/load-card! store "nonexistent"))))))--(deftest caching-returns-same-object-test- (let [store (make-test-store)]- (testing "loading the same entity twice returns the cached version"- (let [first-load (store/load-database! store "DB")- second-load (store/load-database! store "DB")]- (is (identical? first-load second-load))))))--(deftest cached-entity-test- (let [store (make-test-store)]- (testing "cached-entity returns nil before loading"- (is (nil? (store/cached-entity store :database "DB"))))- (testing "cached-entity returns data after loading"- (store/load-database! store "DB")- (is (some? (store/cached-entity store :database "DB"))))))--(deftest load-table-assigns-database-id-test- (let [store (make-test-store)]- (testing "loading a table also assigns an ID to its database"- (store/load-table! store ["DB" "PUBLIC" "ORDERS"])- (is (some? (store/ref->id store :database "DB"))- "Database should have an assigned ID after loading a table"))))--(deftest load-field-assigns-table-id-test- (let [store (make-test-store)]- (testing "loading a field also assigns an ID to its table"- (store/load-field! store ["DB" "PUBLIC" "ORDERS" "ID"])- (is (some? (store/ref->id store :table ["DB" "PUBLIC" "ORDERS"]))- "Table should have an assigned ID after loading a field"))))diff --git a/enterprise/backend/test/metabase_enterprise/checker/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/checker/test_helpers.cljdeleted file mode 100644index dcc3c8ca4cc..00000000000--- a/enterprise/backend/test/metabase_enterprise/checker/test_helpers.clj+++ /dev/null@@ -1,68 +0,0 @@-(ns metabase-enterprise.checker.test-helpers- "Shared test helpers for the checker module."- (:require- [metabase-enterprise.checker.source :as source]))--(defn make-schema-source- "Create an in-memory SchemaSource from maps of databases, tables, and fields."- [{:keys [databases tables fields]}]- (let [fields-by-table (reduce-kv (fn [m field-path _]- (let [table-path (subvec field-path 0 3)]- (update m table-path (fnil conj #{}) field-path)))- {}- fields)]- (reify- source/SchemaSource- (resolve-database [_ db-name] (get databases db-name))- (resolve-table [_ table-path] (get tables table-path))- (resolve-field [_ field-path] (get fields field-path))- (fields-for-table [_ table-path] (get fields-by-table table-path))- (all-field-paths [_] (keys fields))- (all-database-names [_] (keys databases))- (all-table-paths [_] (keys tables))- (tables-for-database [_ db-name] (filterv #(= (first %) db-name) (keys tables))))))--(defn make-assets-source- "Create an in-memory AssetsSource from maps of entities.- Supports :cards, :snippets, :transforms, :segments, :dashboards,- :collections, :documents, :measures."- [{:keys [cards snippets transforms segments dashboards collections documents measures]}]- (reify- source/AssetsSource- (resolve-card [_ entity-id] (get cards entity-id))- (resolve-snippet [_ entity-id] (get snippets entity-id))- (resolve-transform [_ entity-id] (get transforms entity-id))- (resolve-segment [_ entity-id] (get segments entity-id))- (resolve-dashboard [_ entity-id] (get dashboards entity-id))- (resolve-collection [_ entity-id] (get collections entity-id))- (resolve-document [_ entity-id] (get documents entity-id))- (resolve-measure [_ entity-id] (get measures entity-id))))--(defn make-schema-index- "Create a file index for an in-memory schema source.- Values are :memory since resolution goes through the source, not files."- [{:keys [databases tables fields]}]- {:database (zipmap (keys databases) (repeat :memory))- :table (zipmap (keys tables) (repeat :memory))- :field (zipmap (keys fields) (repeat :memory))})--(defn make-assets-index- "Create a file index for an in-memory assets source.- Values are :memory since resolution goes through the source, not files."- [{:keys [cards snippets transforms segments dashboards collections documents measures]}]- (cond-> {:card (zipmap (keys cards) (repeat :memory))}- snippets (assoc :snippet (zipmap (keys snippets) (repeat :memory)))- transforms (assoc :transform (zipmap (keys transforms) (repeat :memory)))- segments (assoc :segment (zipmap (keys segments) (repeat :memory)))- dashboards (assoc :dashboard (zipmap (keys dashboards) (repeat :memory)))- collections (assoc :collection (zipmap (keys collections) (repeat :memory)))- documents (assoc :document (zipmap (keys documents) (repeat :memory)))- measures (assoc :measure (zipmap (keys measures) (repeat :memory)))))--(defn make-sources-and-index- "Convenience: split an entities map into schema-source, assets-source, and merged index.- Returns [schema-source assets-source index]."- [entities]- [(make-schema-source entities)- (make-assets-source entities)- (merge (make-schema-index entities) (make-assets-index entities))])diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.cljindex a9f24dac432..27952cf7210 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj@@ -28,8 +28,6 @@(reify source.p/Source(create-branch [_ _branch _base]nil)- (delete-branch [_ _branch]- nil)(branches [_](if error-on-branches?(throw (Exception. "Repository not found"))@@ -1498,7 +1496,6 @@(reify source.p/Source(branches [_] ["main"])(create-branch [_ _ _] nil)- (delete-branch [_ _] nil)(default-branch [_] "main")(snapshot [_](throw (ex-info (str "Invalid branch: " branch)@@ -1559,7 +1556,6 @@(let [failing-source (reify source.p/Source(branches [_] ["main"])(create-branch [_ _ _] nil)- (delete-branch [_ _] nil)(default-branch [_] "main")(snapshot [_] (throw (RuntimeException. "boom")))(snapshot-at [_ _version] (throw (RuntimeException. "boom"))))]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex 3a7e735f884..bdb1bfcd10c 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -1802,7 +1802,6 @@ serdes/meta:(reify source.p/Source(branches [_] ["main"])(create-branch [_ _ _] nil)- (delete-branch [_ _] nil)(default-branch [_] "main")(snapshot [_] (export-test-snapshot "remote-R"))(snapshot-at [_ v] (export-test-snapshot v))))@@ -1887,7 +1886,6 @@ serdes/meta:no-resolve-source (reify source.p/Source(branches [_] ["main"])(create-branch [_ _ _] nil)- (delete-branch [_ _] nil)(default-branch [_] "main")(snapshot [_] (export-test-snapshot "remote-R"))(snapshot-at [_ _] nil))]@@ -2005,7 +2003,6 @@ serdes/meta:(let [no-base-source (reify source.p/Source(branches [_] ["main"])(create-branch [_ _ _] nil)- (delete-branch [_ _] nil)(default-branch [_] "main")(snapshot [_] (export-test-snapshot "remote-R"))(snapshot-at [_ _] nil))]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.cljindex 2c20689bf38..d346abb3cb2 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj@@ -160,16 +160,6 @@(git-working-create-branch! _remote "branch-3")(is (= ["branch-1" "branch-2" "branch-3" "master"] (source.p/branches source))))))-(deftest delete-branch-test- (mt/with-temp-dir [remote-dir nil]- (let [[source _remote] (init-source! "master" remote-dir :branches ["branch-1" "branch-2"])]- (testing "deletes the branch on the remote"- (source.p/delete-branch source "branch-1")- (is (= ["branch-2" "master"] (source.p/branches source))))- (testing "deleting a branch that does not exist is a no-op"- (source.p/delete-branch source "no-such-branch")- (is (= ["branch-2" "master"] (source.p/branches source)))))))-(deftest snapshot(mt/with-temp-dir [remote-dir nil](let [[master _remote] (init-source! "master" remote-dirdiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.cljindex 609f4ce286e..31895f28fe7 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj@@ -23,9 +23,6 @@(create-branch [_ _branch _base]nil)- (delete-branch [_ _branch]- nil)-(branches [_]["main"])diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex 52e95d24b7e..1b1e3325c39 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -234,10 +234,6 @@ width: fixed(create-branch [_this branch _base](swap! branches-atom conj [branch (str branch "-ref")]))- (delete-branch [_this branch]- (swap! branches-atom (fn [branches] (vec (remove #(= (first %) branch) branches))))- nil)-(branches [_this](case fail-mode:branches-error (throw (java.net.UnknownHostException. "Network error"))@@ -346,7 +342,6 @@ width: fixed(reify source.p/Source(branches [_] [branch])(create-branch [_ _ _] nil)- (delete-branch [_ _] nil)(default-branch [_] branch)(snapshot [_] (mk-snapshot (:current @state)))(snapshot-at [_ v] (when (contains? (:trees @state) v) (mk-snapshot v))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex 9a2db8279bf..626463de9fa 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -4,23 +4,19 @@[clojure.string :as str][clojure.test :refer :all][clojure.walk :as walk]- [medley.core :as m][metabase-enterprise.serialization.api :as api.serialization]- [metabase-enterprise.serialization.metadata-file-import :as metadata-file-import][metabase-enterprise.serialization.v2.extract :as v2.extract][metabase-enterprise.serialization.v2.ingest :as v2.ingest][metabase.analytics.snowplow-test :as snowplow-test][metabase.models.serialization :as serdes]- [metabase.permissions.core :as perms][metabase.search.core :as search][metabase.search.test-util :as search.tu][metabase.test :as mt][metabase.util.compress :as u.compress]- [metabase.util.json :as json][metabase.util.random :as u.random][toucan2.core :as t2])(:import- (java.io ByteArrayInputStream File)+ (java.io File)(org.apache.commons.compress.archivers.tar TarArchiveEntry TarArchiveInputStream)(org.apache.commons.compress.compressors.gzip GzipCompressorInputStream)))@@ -498,373 +494,6 @@{:request-options {:headers {"content-type" "multipart/form-data"}}}{:file (byte-array 0)})))))))-(deftest metadata-export-basic-test- (testing "POST /api/ee/serialization/metadata/export — happy path with one db/table/field"- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {t-id :id t-name :name} {:db_id db-id :schema "PUBLIC"- :description "A test table"}- :model/Field {f-id :id f-name :name} {:table_id t-id- :base_type :type/Integer- :database_type "BIGINT"- :semantic_type :type/PK}]- (let [{:keys [databases tables fields]} (mt/user-http-request :crowberto :post 202- "ee/serialization/metadata/export"- :with-databases true- :with-tables true- :with-fields true)- test-db (m/find-first (comp #{db-id} :id) databases)- test-table (m/find-first (comp #{t-id} :id) tables)- test-field (m/find-first (comp #{f-id} :id) fields)]- (is (=? {:id db-id :name db-name :engine "h2"}- test-db))- (is (=? {:id t-id- :db_id db-id- :name t-name- :schema "PUBLIC"- :description "A test table"}- test-table))- (is (=? {:id f-id- :table_id t-id- :name f-name- :base_type "type/Integer"- :database_type "BIGINT"- :semantic_type "type/PK"}- test-field)))))))--(deftest metadata-export-optional-properties-test- (testing "POST /api/ee/serialization/metadata/export — effective_type, coercion_strategy and description are emitted when set"- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}- :model/Field {f-id :id} {:table_id t-id- :base_type :type/Text- :database_type "VARCHAR"- :effective_type :type/DateTime- :coercion_strategy :Coercion/ISO8601->DateTime- :description "When this happened"}]- (let [{:keys [fields]} (mt/user-http-request :crowberto :post 202- "ee/serialization/metadata/export"- :with-fields true)- test-field (m/find-first (comp #{f-id} :id) fields)]- (is (=? {:id f-id- :base_type "type/Text"- :effective_type "type/DateTime"- :coercion_strategy "Coercion/ISO8601->DateTime"- :description "When this happened"}- test-field)))))))--(deftest metadata-export-parent-field-test- (testing "POST /api/ee/serialization/metadata/export — parent_id is emitted only when raw field.parent_id is set; nfc_path is decoded"- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}- :model/Field {root-id :id} {:table_id t-id- :base_type :type/Text}- :model/Field {parent-id :id} {:table_id t-id- :name "data"- :base_type :type/Dictionary}- :model/Field {child-id :id} {:table_id t-id- :name "city"- :base_type :type/Text- :parent_id parent-id- :nfc_path ["data" "city"]}]- (let [{:keys [fields]} (mt/user-http-request :crowberto :post 202- "ee/serialization/metadata/export"- :with-fields true)- root-field (m/find-first (comp #{root-id} :id) fields)- child-field (m/find-first (comp #{child-id} :id) fields)]- (is (=? {:id root-id :table_id t-id} root-field))- (is (not (contains? root-field :parent_id)))- (is (=? {:id child-id- :table_id t-id- :parent_id parent-id- :nfc_path ["data" "city"]}- child-field)))))))--(deftest metadata-export-fk-test- (testing "POST /api/ee/serialization/metadata/export — fk_target_field_id is emitted as a raw id"- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}- :model/Field {pk-id :id} {:table_id t-id- :base_type :type/Integer- :semantic_type :type/PK}- :model/Field {fk-id :id} {:table_id t-id- :base_type :type/Integer- :semantic_type :type/FK- :fk_target_field_id pk-id}]- (let [{:keys [fields]} (mt/user-http-request :crowberto :post 202- "ee/serialization/metadata/export"- :with-fields true)- pk-field (m/find-first (comp #{pk-id} :id) fields)- fk-field (m/find-first (comp #{fk-id} :id) fields)]- (is (=? {:id pk-id :table_id t-id :semantic_type "type/PK"}- pk-field))- (is (=? {:id fk-id- :table_id t-id- :semantic_type "type/FK"- :fk_target_field_id pk-id}- fk-field)))))))--(deftest metadata-export-hidden-table-test- (testing "POST /api/ee/serialization/metadata/export — hidden tables and their fields are excluded"- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {visible-id :id} {:db_id db-id :schema "PUBLIC"}- :model/Table {hidden-id :id} {:db_id db-id :schema "PUBLIC"- :visibility_type :hidden}- :model/Field {visible-f-id :id} {:table_id visible-id- :base_type :type/Integer}- :model/Field {hidden-f-id :id} {:table_id hidden-id- :base_type :type/Integer}]- (let [{:keys [tables fields]} (mt/user-http-request :crowberto :post 202- "ee/serialization/metadata/export"- :with-tables true :with-fields true)]- (is (some (comp #{visible-id} :id) tables))- (is (not-any? (comp #{hidden-id} :id) tables))- (is (some (comp #{visible-f-id} :id) fields))- (is (not-any? (comp #{hidden-f-id} :id) fields)))))))--(deftest metadata-export-inactive-table-test- (testing "POST /api/ee/serialization/metadata/export — inactive tables and all of their fields are excluded"- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"- :active false}- :model/Field {active-f-id :id} {:table_id t-id- :base_type :type/Integer}- :model/Field {inactive-f-id :id} {:table_id t-id- :base_type :type/Integer- :active false}]- (let [{:keys [tables fields]} (mt/user-http-request :crowberto :post 202- "ee/serialization/metadata/export"- :with-tables true :with-fields true)]- (is (not-any? (comp #{t-id} :id) tables))- (is (not-any? (comp #{active-f-id} :id) fields))- (is (not-any? (comp #{inactive-f-id} :id) fields)))))))--(deftest metadata-export-sensitive-field-test- (testing "POST /api/ee/serialization/metadata/export — sensitive fields are excluded"- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}- :model/Field {email-id :id} {:table_id t-id :base_type :type/Text}- :model/Field {secret-id :id} {:table_id t-id :base_type :type/Text- :visibility_type :sensitive}]- (let [{:keys [fields]} (mt/user-http-request :crowberto :post 202- "ee/serialization/metadata/export"- :with-fields true)]- (is (some (comp #{email-id} :id) fields))- (is (not-any? (comp #{secret-id} :id) fields)))))))--(deftest metadata-export-inactive-field-test- (testing "POST /api/ee/serialization/metadata/export — inactive fields are excluded; their active sibling and table still appear"- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"}- :model/Field {active-f-id :id} {:table_id t-id- :base_type :type/Integer}- :model/Field {inactive-f-id :id} {:table_id t-id- :base_type :type/Integer- :active false}]- (let [{:keys [tables fields]} (mt/user-http-request :crowberto :post 202- "ee/serialization/metadata/export"- :with-tables true :with-fields true)]- (is (some (comp #{t-id} :id) tables))- (is (some (comp #{active-f-id} :id) fields))- (is (not-any? (comp #{inactive-f-id} :id) fields)))))))--(deftest metadata-export-db-routing-test- (testing "POST /api/ee/serialization/metadata/export — router (mirror) databases are excluded"- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database _ {:engine :h2}- :model/Database {primary-id :id} {:engine :h2}- :model/Database {mirror-id :id} {:engine :h2- :router_database_id primary-id}]- (let [{:keys [databases]} (mt/user-http-request :crowberto :post 202- "ee/serialization/metadata/export"- :with-databases true)]- (is (some (comp #{primary-id} :id) databases))- (is (not-any? (comp #{mirror-id} :id) databases)))))))--(deftest metadata-export-table-permission-test- (testing "POST /api/ee/serialization/metadata/export — tables the current user can't access are excluded"- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {accessible-id :id} {:db_id db-id :schema "PUBLIC"}- :model/Table {restricted-id :id} {:db_id db-id :schema "PUBLIC"}- :model/Field {ok-f-id :id} {:table_id accessible-id- :base_type :type/Integer}- :model/Field {no-f-id :id} {:table_id restricted-id- :base_type :type/Integer}- :model/PermissionsGroup pg {}]- (perms/add-user-to-group! (mt/user->id :rasta) pg)- (t2/delete! :model/DataPermissions :db_id db-id)- (perms/set-database-permission! pg db-id :perms/view-data :blocked)- (perms/set-database-permission! pg db-id :perms/create-queries :no)- (perms/set-table-permission! pg accessible-id :perms/view-data :unrestricted)- (perms/set-table-permission! pg accessible-id :perms/create-queries :query-builder)- (let [{:keys [tables fields]} (mt/user-http-request :rasta :post 202- "ee/serialization/metadata/export"- :with-tables true :with-fields true)]- (is (some (comp #{accessible-id} :id) tables))- (is (not-any? (comp #{restricted-id} :id) tables))- (is (some (comp #{ok-f-id} :id) fields))- (is (not-any? (comp #{no-f-id} :id) fields)))))))--(deftest metadata-export-token-feature-test- (testing "POST /api/ee/serialization/metadata/export requires the :serialization premium feature"- (mt/with-premium-features #{}- (mt/assert-has-premium-feature-error- "Serialization"- (mt/user-http-request :crowberto :post 402 "ee/serialization/metadata/export")))))--;;; --------------------------------------- /metadata/import -----------------------------------------(defn- import-metadata!- "Helper: JSON-encode `payload` and POST it as an octet-stream body to- `/api/ee/serialization/metadata/import`."- ([payload] (import-metadata! :crowberto 202 payload))- ([user expected-status payload]- (mt/user-http-request- user :post expected-status "ee/serialization/metadata/import"- {:request-options- {:headers {"content-type" "application/octet-stream"}- :body (ByteArrayInputStream.- (.getBytes ^String (json/encode payload) "UTF-8"))}})))--(defn- import-status!- "Helper: GET `/api/ee/serialization/metadata/import/:id`."- ([id] (import-status! :crowberto 200 id))- ([user expected-status id]- (mt/user-http-request- user :get expected-status (str "ee/serialization/metadata/import/" id))))--(deftest metadata-import-roundtrip-test- (testing "POST /api/ee/serialization/metadata/import is the inverse of POST /metadata/export"- ;; The agent runs the import on a separate connection from the test thread;- ;; mt/with-temp's default rollback-only wrapper would hide the temp rows- ;; from that connection. Real commits + auto-sync disabled lets the agent- ;; see what the test set up.- ;; Warm test-data before disabling sync: the :model/Field with-temp default resolves- ;; `(data/id :checkins)`, which needs a synced test-data DB.- (mt/db)- (mt/with-temporary-setting-values [disable-auto-sync true]- (mt/test-helpers-set-global-values!- (mt/with-premium-features #{:serialization}- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC"- :description "round trip"}- :model/Field _ {:table_id t-id- :base_type :type/Integer- :database_type "BIGINT"- :semantic_type :type/PK}]- (let [exported (mt/user-http-request :crowberto :post 202- "ee/serialization/metadata/export"- :with-databases true- :with-tables true- :with-fields true)- before-tables (t2/count :model/Table :db_id db-id)- before-fields (t2/count :model/Field :table_id t-id)- resp (import-metadata! exported)]- (is (true? (:queued resp)) "POST acknowledges the import was queued")- (is (string? (:import-id resp)) "POST returns a string import-id")- ;; The import endpoint hands the file off to an agent and returns- ;; immediately. Block until the agent has drained it, then assert- ;; the agent didn't silently swallow an exception (without this- ;; check, the row-count assertions below would falsely pass on a- ;; failed re-import since nothing would have changed).- (await @#'metadata-file-import/import-agent)- (is (= :ok (:status (metadata-file-import/import-status (:import-id resp))))- "agent finished the import successfully")- (testing "no rows are added by re-importing the same payload"- (is (= before-tables (t2/count :model/Table :db_id db-id)))- (is (= before-fields (t2/count :model/Field :table_id t-id))))- (testing "the table description is preserved"- (is (= "round trip"- (t2/select-one-fn :description :model/Table t-id)))))))))))--(deftest metadata-import-post-returns-202-and-import-id-test- (testing "POST /api/ee/serialization/metadata/import returns 202 with :queued true and a string :import-id"- (mt/with-premium-features #{:serialization}- (let [resp (import-metadata! :crowberto 202 {:databases [] :tables [] :fields []})]- (is (true? (:queued resp)))- (is (string? (:import-id resp)) ":import-id is a string")- (is (uuid? (java.util.UUID/fromString (:import-id resp)))- ":import-id parses as a UUID")- (await @#'metadata-file-import/import-agent)))))--(deftest metadata-import-status-endpoint-lifecycle-test- (testing "GET /metadata/import/:id reports the import status across its lifecycle"- (mt/with-temporary-setting-values [disable-auto-sync true]- (mt/test-helpers-set-global-values!- (mt/with-premium-features #{:serialization}- (let [resp (import-metadata! :crowberto 202 {:databases [] :tables [] :fields []})- id (:import-id resp)]- ;; settle the agent, then the GET endpoint should report a terminal status- (await @#'metadata-file-import/import-agent)- (let [status (import-status! :crowberto 200 id)]- (is (= id (:id status)) "the status response echoes the id")- (is (string? (:status status)) ":status is serialized as a string")- (is (contains? #{"ok" "error"} (:status status))- "after the agent settles the status is terminal")- (is (contains? status :enqueued-at))- (is (contains? status :started-at))- (is (contains? status :finished-at))- (is (contains? status :wall-ms)))))))))--(deftest metadata-import-status-omits-file-path-test- (testing "GET /metadata/import/:id never leaks the server-side temp file path"- (mt/with-premium-features #{:serialization}- (let [resp (import-metadata! :crowberto 202 {:databases [] :tables [] :fields []})- id (:import-id resp)]- (await @#'metadata-file-import/import-agent)- (let [status (import-status! :crowberto 200 id)]- (is (not (contains? status :file))- "the wire shape must not include :file")- (is (= #{:id :status :enqueued-at :started-at :finished-at :wall-ms :error}- (set (keys status)))- "the wire shape is exactly the documented key set"))))))--(deftest metadata-import-status-unknown-id-404-test- (testing "GET /metadata/import/:id returns 404 for an unknown or evicted id"- (mt/with-premium-features #{:serialization}- (import-status! :crowberto 404 (str (java.util.UUID/randomUUID))))))--(deftest metadata-import-status-superuser-test- (testing "GET /metadata/import/:id — non-admins get a 403"- (mt/with-premium-features #{:serialization}- (is (= "You don't have permissions to do that."- (import-status! :rasta 403 (str (java.util.UUID/randomUUID))))))))--(deftest metadata-import-status-token-feature-test- (testing "GET /metadata/import/:id requires the :serialization premium feature"- (mt/with-premium-features #{}- (mt/assert-has-premium-feature-error- "Serialization"- (import-status! :crowberto 402 (str (java.util.UUID/randomUUID)))))))--(deftest metadata-import-wrong-content-type-test- (testing "POST /api/ee/serialization/metadata/import — JSON content-type is rejected with 415"- (mt/with-premium-features #{:serialization}- (mt/user-http-request :crowberto :post 415- "ee/serialization/metadata/import"- {:tables [] :fields []}))))--(deftest metadata-import-superuser-test- (testing "POST /api/ee/serialization/metadata/import — non-admins get a 403"- (mt/with-premium-features #{:serialization}- (is (= "You don't have permissions to do that."- (import-metadata! :rasta 403 {:tables [] :fields []}))))))--(deftest metadata-import-token-feature-test- (testing "POST /api/ee/serialization/metadata/import requires the :serialization premium feature"- (mt/with-premium-features #{}- (mt/assert-has-premium-feature-error- "Serialization"- (import-metadata! :crowberto 402 {:tables [] :fields []})))))-(deftest serialization-cleanup-test(testing "No temp files are left behind after export/import operations"(let [known-files (set (.list (io/file api.serialization/parent-dir)))]diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/concurrency_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/concurrency_test.cljdeleted file mode 100644index 9a2e81e3f9f..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/concurrency_test.clj+++ /dev/null@@ -1,259 +0,0 @@-(ns metabase-enterprise.serialization.metadata-file-import.concurrency-test- "Tests for the in-JVM concurrency guard on metadata-file-import. The guard- is a single Clojure agent serializing all imports plus a busy-predicate- registered with `metabase.sync.util` so that warehouse-sync skips with a- WARN log line whenever an import is in flight."- (:require- [clojure.test :refer :all]- [metabase-enterprise.serialization.metadata-file-import :as mfi]- [metabase.sync.util :as sync.util]- [metabase.test :as mt]- [metabase.util.json :as json])- (:import- (java.io File)))--(set! *warn-on-reflection* true)--;; Kondo flags `test-helpers-set-global-values!` inside any fixture as-;; "destructive in parallel context". Here it is *intentionally* the opt-out-;; switch — without it the agent runs on a separate connection that can't see-;; the test's mt/with-temp rollback-only state.-#_{:clj-kondo/ignore [:metabase/validate-deftest]}-(use-fixtures :once- (fn [thunk]- ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with- ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.- (mt/db)- (mt/with-temporary-setting-values [disable-auto-sync true]- (mt/test-helpers-set-global-values!- (thunk)))))--(defn- tiny-meta-file ^File [data]- (let [f (File/createTempFile "mfi-concurrency-" ".json")]- (.deleteOnExit f)- (spit f (json/encode data))- f))--(defn- tiny-payload [db-name]- {:databases [{:id 7 :name db-name :engine "postgres"}]- :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]- :fields [{:id 1000 :table_id 100 :name "x"- :base_type "type/Integer" :database_type "integer"}]})--;;; ============================== import-running? ==============================--(deftest import-running-false-when-idle-test- (testing "import-running? returns false when no import is in flight"- (await @#'mfi/import-agent)- (is (false? (mfi/import-running?)))))--(deftest import-running-toggles-around-enqueue-test- (testing "import-running? is true mid-flight, false after the agent settles"- (mt/with-temp [:model/Database {} {:name "mfi-running-toggles" :engine :postgres}]- (let [file (tiny-meta-file (tiny-payload "mfi-running-toggles"))- observed (atom nil)- real-import mfi/import-metadata-file!]- (with-redefs [mfi/import-metadata-file!- (fn [f] (reset! observed (mfi/import-running?))- (real-import f))]- (mfi/enqueue-import! file)- (await @#'mfi/import-agent))- (is (true? @observed)- "import-running? is true while the agent is inside the import fn")- (is (false? (mfi/import-running?))- "import-running? is false again once the agent finishes")))))--;;; ============================== Serialization ==============================--(deftest two-enqueues-run-sequentially-test- (testing "two enqueued imports execute one-at-a-time, in arrival order"- (mt/with-temp [:model/Database {} {:name "mfi-seq-1" :engine :postgres}- :model/Database {} {:name "mfi-seq-2" :engine :postgres}]- (let [f1 (tiny-meta-file (tiny-payload "mfi-seq-1"))- f2 (tiny-meta-file (tiny-payload "mfi-seq-2"))- events (atom [])- real-import mfi/import-metadata-file!]- (with-redefs [mfi/import-metadata-file!- (fn [^java.io.File f]- (swap! events conj [:start (.getName f) (System/nanoTime)])- (real-import f)- (swap! events conj [:end (.getName f) (System/nanoTime)]))]- (mfi/enqueue-import! f1)- (mfi/enqueue-import! f2)- (await @#'mfi/import-agent))- (let [evs @events]- (is (= 4 (count evs)) "two starts + two ends")- (is (= :start (first (nth evs 0))))- (is (= :end (first (nth evs 1))))- (is (= :start (first (nth evs 2))))- (is (= :end (first (nth evs 3))))- (is (= (second (nth evs 0)) (second (nth evs 1)))- "first import's start and end are for the same file")- (is (= (second (nth evs 2)) (second (nth evs 3)))- "second import's start and end are for the same file")- (is (<= (nth (nth evs 1) 2) (nth (nth evs 2) 2))- "second import does not start until first has finished"))))))--;;; ============================== Agent does not poison ==============================--(deftest agent-survives-failing-import-test- (testing "an exception inside the import body does not leave the agent in a failed state"- (await @#'mfi/import-agent)- (with-redefs [mfi/import-metadata-file!- (fn [_f] (throw (ex-info "synthetic" {:kind :test-failure})))]- (mfi/enqueue-import! (tiny-meta-file (tiny-payload "mfi-fail")))- (await @#'mfi/import-agent))- (is (nil? (agent-error @#'mfi/import-agent))- "agent did not enter failed state")- (is (false? (mfi/import-running?))- "state reset to :idle after the failure")))--;;; ============================== Registry ==============================--(deftest enqueue-import-returns-string-id-and-creates-queued-record-test- (testing "enqueue-import! returns a string UUID id and synchronously inserts a :queued record"- (await @#'mfi/import-agent)- ;; Block the agent so we can observe the :queued record before it transitions.- (let [release (promise)]- (with-redefs [mfi/import-metadata-file! (fn [_f] @release :ok)]- (let [id (mfi/enqueue-import! (tiny-meta-file (tiny-payload "mfi-queued")))]- (is (string? id) "enqueue-import! returns a string id, not :queued")- (is (uuid? (java.util.UUID/fromString id))- "the returned id parses as a UUID")- (let [record (mfi/import-status id)]- (is (some? record) "a registry record exists for the returned id")- (is (= id (:id record)))- (is (contains? #{:queued :running} (:status record))- "record is :queued (or already :running if the agent picked it up)")- (is (some? (:enqueued-at record)) ":enqueued-at is set on insert"))- (deliver release :go)- (await @#'mfi/import-agent))))))--(deftest record-transitions-queued-running-ok-test- (testing "the registry record moves :queued -> :running -> :ok through the agent"- (mt/with-temp [:model/Database {} {:name "mfi-lifecycle-ok" :engine :postgres}]- (await @#'mfi/import-agent)- (let [file (tiny-meta-file (tiny-payload "mfi-lifecycle-ok"))- id (mfi/enqueue-import! file)]- ;; Capturing the record mid-flight is racy; settle the agent and assert- ;; the terminal record — the recorded timestamps prove it passed- ;; through :running.- (await @#'mfi/import-agent)- (let [record (mfi/import-status id)]- (is (some? record))- (is (= :ok (:status record)) "terminal status is :ok on success")- (is (some? (:started-at record)) ":started-at set => passed through :running")- (is (some? (:finished-at record)) ":finished-at set on completion")- (is (number? (:wall-ms record)) ":wall-ms recorded on completion")- (is (nil? (:error record)) "no :error on a successful import"))))))--(deftest record-transitions-to-error-on-failing-import-test- (testing "a failing import yields an :error record and does not poison the agent"- (await @#'mfi/import-agent)- (let [id (with-redefs [mfi/import-metadata-file!- (fn [_f] (throw (ex-info "synthetic failure" {:kind :test-failure})))]- (let [id (mfi/enqueue-import! (tiny-meta-file (tiny-payload "mfi-lifecycle-err")))]- (await @#'mfi/import-agent)- id))]- (is (nil? (agent-error @#'mfi/import-agent))- "agent did not enter a failed state")- (let [record (mfi/import-status id)]- (is (some? record))- (is (= :error (:status record)) "terminal status is :error on failure")- (is (some? (:started-at record)))- (is (some? (:finished-at record)))- (is (number? (:wall-ms record)))- (is (string? (:error record)) ":error holds a stringified exception")))))--(deftest import-status-nil-for-unknown-id-test- (testing "import-status returns nil for an id that was never enqueued"- (is (nil? (mfi/import-status (str (java.util.UUID/randomUUID))))- "unknown id => nil")- (is (nil? (mfi/import-status "not-even-a-uuid"))- "garbage id => nil")))--;;; ============================== Eviction ==============================--(defn- terminal-record [i]- [(str "id-" i) {:id (str "id-" i) :status :ok- :finished-at (java.time.Instant/ofEpochSecond i)}])--(deftest prune-terminal-records-noop-under-bound-test- (testing "prune-terminal-records leaves the registry untouched when terminal count is within the bound"- (let [limit @#'mfi/terminal-record-limit- reg (into {} (map terminal-record) (range (dec limit)))]- (is (= reg (#'mfi/prune-terminal-records reg))))))--(deftest prune-terminal-records-drops-oldest-terminal-test- (testing "prune-terminal-records caps terminal records at the bound, dropping the oldest by :finished-at"- (let [limit @#'mfi/terminal-record-limit- extra 5- reg (into {} (map terminal-record) (range (+ limit extra)))- pruned (#'mfi/prune-terminal-records reg)]- (is (= limit (count pruned))- "terminal records are capped at the bound")- (is (every? #(contains? pruned (str "id-" %)) (range extra (+ limit extra)))- "the most-recent `limit` records are retained")- (is (not-any? #(contains? pruned (str "id-" %)) (range extra))- "the oldest records are evicted"))))--(deftest prune-terminal-records-never-drops-in-flight-test- (testing ":queued and :running records are never pruned, regardless of count"- (let [limit @#'mfi/terminal-record-limit- in-flight (into {} (for [i (range (* 3 limit))]- [(str "q-" i) {:id (str "q-" i)- :status (if (even? i) :queued :running)}]))- terminal (into {} (map terminal-record) (range (* 2 limit)))- pruned (#'mfi/prune-terminal-records (merge in-flight terminal))]- (is (every? #(contains? pruned (str "q-" %)) (range (* 3 limit)))- "every in-flight record survives")- (is (= limit (count (filter (comp #{:ok} :status val) pruned)))- "terminal records are still capped at the bound"))))--;;; ============================== Sync-side skip ==============================--(deftest sync-skips-when-import-is-running-test- (testing "sync.util/do-sync-operation skips its body when an import is in flight"- (mt/with-temp [:model/Database db {:name "mfi-sync-skip" :engine :postgres}]- (await @#'mfi/import-agent)- ;; Drive a real :running record into the registry by blocking the agent- ;; inside a redef'd slow import, then exercise the busy-check while it sits.- (let [release (promise)]- (with-redefs [mfi/import-metadata-file! (fn [_f] @release :ok)]- (mfi/enqueue-import! (tiny-meta-file (tiny-payload "mfi-sync-skip")))- ;; Wait until the agent has actually flipped a record to :running.- (let [deadline (+ (System/currentTimeMillis) 5000)]- (while (and (not (mfi/import-running?))- (< (System/currentTimeMillis) deadline))- (Thread/sleep 10)))- (try- (is (true? (mfi/import-running?))- "an import is in flight before we exercise the busy-check")- (let [sync-ran? (atom false)]- (sync.util/do-sync-operation- :sync-metadata- db- "test-sync"- (fn [] (reset! sync-ran? true) :result-not-checked))- (is (false? @sync-ran?)- "sync body did NOT run because import-busy short-circuited it"))- (finally- (deliver release :go)- (await @#'mfi/import-agent))))))))--(deftest sync-runs-normally-when-no-import-in-flight-test- (testing "control: sync body executes normally when no import is in flight"- ;; The namespace fixture sets `disable-auto-sync true`, which itself makes- ;; `should-sync?` short-circuit before the busy-check runs. Re-enable it- ;; locally so we're actually exercising the busy-check path.- (mt/with-temporary-setting-values [disable-auto-sync false]- (mt/with-temp [:model/Database db {:name "mfi-sync-control" :engine :postgres}]- (await @#'mfi/import-agent)- (let [sync-ran? (atom false)]- (sync.util/do-sync-operation- :sync-metadata- db- "test-sync-control"- (fn [] (reset! sync-ran? true) :result-not-checked))- (is (true? @sync-ran?)- "sync body DID run because no import is busy"))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/depth_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/depth_test.cljdeleted file mode 100644index 805ad21befe..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/depth_test.clj+++ /dev/null@@ -1,205 +0,0 @@-(ns metabase-enterprise.serialization.metadata-file-import.depth-test- "Tests for `compute-staging-depth!` — the post-drain step that tags every- `metabase_field_import` row with a `depth` value, where 0 = root and depth- d = max(parent depth, fk_target depth) + 1.-- Pre-condition for the tagging step is that orphan refs have already been- rejected by `assert-no-orphan-refs!`. These tests assume that condition- holds (every fixture below is consistent)."- (:require- [clojure.test :refer :all]- [metabase-enterprise.serialization.metadata-file-import.processors :as p]- [toucan2.core :as t2]))--(defn- insert-staging-fields!- "Insert a sequence of partial field rows into `metabase_field_import`,- filling in required NOT NULL columns with sensible defaults."- [rows]- (let [defaults {:name "x" :base_type "type/Integer" :database_type "int"}]- (t2/insert! :metabase_field_import- (mapv (fn [row] (merge defaults {:source_table_id 1} row)) rows))))--(defn- depth-by-source-id- "Return `{source_id → depth}` after tagging."- []- (into {}- (map (juxt :source_id :depth))- (t2/query "SELECT source_id, depth FROM metabase_field_import ORDER BY source_id")))--;;; ============================== empty + roots ==============================--(deftest empty-staging-is-noop-test- (try- (p/clear-staging-tables!)- (is (= 0 (p/compute-staging-depth!)))- (finally (p/clear-staging-tables!))))--(deftest all-roots-get-depth-zero-test- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 1}- {:source_id 2}- {:source_id 3}])- (is (= 0 (p/compute-staging-depth!))- "max depth assigned is 0 when all rows are roots")- (is (= {1 0, 2 0, 3 0} (depth-by-source-id)))- (finally (p/clear-staging-tables!))))--;;; ============================== parent chains ==============================--(deftest one-level-parent-chain-test- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 10}- {:source_id 11 :source_parent_id 10}])- (is (= 1 (p/compute-staging-depth!)))- (is (= {10 0, 11 1} (depth-by-source-id)))- (finally (p/clear-staging-tables!))))--(deftest three-level-parent-chain-test- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 100}- {:source_id 101 :source_parent_id 100}- {:source_id 102 :source_parent_id 101}- {:source_id 103 :source_parent_id 102}])- (is (= 3 (p/compute-staging-depth!)))- (is (= {100 0, 101 1, 102 2, 103 3} (depth-by-source-id)))- (finally (p/clear-staging-tables!))))--;;; ============================== fk_target chains ==============================--(deftest fk-target-chain-test- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 200}- {:source_id 201 :source_fk_target_id 200}- {:source_id 202 :source_fk_target_id 201}])- (is (= 2 (p/compute-staging-depth!)))- (is (= {200 0, 201 1, 202 2} (depth-by-source-id)))- (finally (p/clear-staging-tables!))))--;;; ============================== mixed (max-depth) semantics ==============================--(deftest depth-is-max-of-parent-and-fk-target-depths-test- (try- (p/clear-staging-tables!)- ;; Build:- ;; A (root) — depth 0- ;; B → A (parent) — depth 1- ;; C → B (parent) — depth 2- ;; D → A (parent, depth 1) AND fk_target → C (depth 2)- ;; → D should be depth 3, not 2 (max of 1, 2 + 1)- (insert-staging-fields! [{:source_id 10}- {:source_id 20 :source_parent_id 10}- {:source_id 30 :source_parent_id 20}- {:source_id 40 :source_parent_id 10 :source_fk_target_id 30}])- (is (= 3 (p/compute-staging-depth!)))- (is (= {10 0, 20 1, 30 2, 40 3} (depth-by-source-id))- "row with parent at depth 1 and fk_target at depth 2 lands at depth 3")- (finally (p/clear-staging-tables!))))--;;; ============================== unfolded leaves ==============================--(deftest unfolded-leaves-are-roots-test- (try- (p/clear-staging-tables!)- ;; Leaves with no source_parent_id (the leaf isn't materialized as a- ;; separate parent storage row) are roots.- (insert-staging-fields! [{:source_id 50}- {:source_id 51}])- (is (= 0 (p/compute-staging-depth!)))- (is (= {50 0, 51 0} (depth-by-source-id)))- (finally (p/clear-staging-tables!))))--;;; ============================== disconnected components ==============================--(deftest multiple-disconnected-chains-tag-in-parallel-test- (try- (p/clear-staging-tables!)- ;; Two unrelated chains.- (insert-staging-fields! [{:source_id 100} ; root in chain A- {:source_id 101 :source_parent_id 100} ; depth 1 chain A- {:source_id 200} ; root in chain B- {:source_id 201 :source_parent_id 200} ; depth 1 chain B- {:source_id 202 :source_parent_id 201}]) ; depth 2 chain B- (is (= 2 (p/compute-staging-depth!)))- (is (= {100 0, 101 1, 200 0, 201 1, 202 2} (depth-by-source-id)))- (finally (p/clear-staging-tables!))))--;;; ============================== cycle detection ==============================--(deftest two-row-parent-cycle-throws-test- (try- (p/clear-staging-tables!)- ;; X.parent = Y, Y.parent = X — neither is a root, neither can ever- ;; be tagged.- (insert-staging-fields! [{:source_id 1 :source_parent_id 2}- {:source_id 2 :source_parent_id 1}])- (let [thrown (try (p/compute-staging-depth!) nil- (catch clojure.lang.ExceptionInfo e e))]- (is (some? thrown) "should have thrown")- (is (= :cycle-in-field-graph (:kind (ex-data thrown))))- (is (= 2 (:remaining-rows-count (ex-data thrown))))- (is (= 2 (count (:remaining-rows-sample (ex-data thrown))))))- (finally (p/clear-staging-tables!))))--(deftest self-referential-parent-id-throws-test- ;; A row pointing at itself can never be tagged: it would need depth d- ;; where its own depth is < d.- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 5 :source_parent_id 5}])- (let [thrown (try (p/compute-staging-depth!) nil- (catch clojure.lang.ExceptionInfo e e))]- (is (some? thrown))- (is (= :cycle-in-field-graph (:kind (ex-data thrown))))- (is (= 1 (:remaining-rows-count (ex-data thrown)))))- (finally (p/clear-staging-tables!))))--(deftest cycle-via-fk-target-throws-test- ;; X.fk_target = Y, Y.fk_target = X — same problem via the FK relationship- ;; rather than parent.- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 1 :source_fk_target_id 2}- {:source_id 2 :source_fk_target_id 1}])- (let [thrown (try (p/compute-staging-depth!) nil- (catch clojure.lang.ExceptionInfo e e))]- (is (some? thrown))- (is (= :cycle-in-field-graph (:kind (ex-data thrown)))))- (finally (p/clear-staging-tables!))))--(deftest cycle-with-some-tagged-rows-test- ;; Two roots and a 2-cycle. The roots get depth 0; the cycle never- ;; resolves. The error data should reflect just the cycle rows.- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 1} ; root- {:source_id 2} ; root- {:source_id 100 :source_parent_id 101} ; cycle- {:source_id 101 :source_parent_id 100}]) ; cycle- (let [thrown (try (p/compute-staging-depth!) nil- (catch clojure.lang.ExceptionInfo e e))]- (is (= :cycle-in-field-graph (:kind (ex-data thrown))))- (is (= 2 (:remaining-rows-count (ex-data thrown)))))- (finally (p/clear-staging-tables!))))--;;; ============================== idempotence ==============================--(deftest depth-tagging-is-idempotent-test- ;; Running twice on the same data should produce the same result.- ;; Useful contract: the merge phase shouldn't have to worry about whether- ;; depths are stale.- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 10}- {:source_id 11 :source_parent_id 10}- {:source_id 12 :source_parent_id 11}])- (let [first-result (p/compute-staging-depth!)- first-depths (depth-by-source-id)- second-result (p/compute-staging-depth!)- second-depths (depth-by-source-id)]- (is (= first-result second-result))- (is (= first-depths second-depths)))- (finally (p/clear-staging-tables!))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/drain_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/drain_test.cljdeleted file mode 100644index e3632eb74e4..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/drain_test.clj+++ /dev/null@@ -1,308 +0,0 @@-(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.drain-test- "Tests for the drain phase of the metadata file importer. Exercises- `process-databases!` and the per-batch drain handlers directly against- hand-built batches — bypassing the parser and orchestrator — and asserts- the resulting staging-table contents.-- The production drain has two implementations: JDBC `executeBatch` for any- appdb (the portable path) and PG `CopyManager` for PostgreSQL (the perf- path). Per-batch tests below use the JDBC variant via the helpers- [[drain-tables!]] / [[drain-fields!]] (each opens its own connection +- PreparedStatement). The drain-paths-equivalence test below additionally- asserts the PG-COPY variant produces the same staging-table state."- (:require- [clojure.test :refer :all]- [metabase-enterprise.serialization.metadata-file-import.processors :as p]- [metabase.app-db.core :as mdb]- [metabase.test :as mt]- [metabase.util.json :as json]- [toucan2.core :as t2])- (:import- (java.sql Connection PreparedStatement)- (org.postgresql PGConnection)- (org.postgresql.copy CopyIn)))--(set! *warn-on-reflection* true)--(use-fixtures :once- (fn [thunk]- ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with- ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.- (mt/db)- (mt/with-temporary-setting-values [disable-auto-sync true]- (thunk))))--(defn- staging-tables- "Return the contents of `metabase_table_import` ordered by `source_id`.- HoneySQL (not raw SQL) so the `schema` column is dialect-quoted correctly- on MySQL/MariaDB, where `SCHEMA` is a reserved word."- []- (t2/query {:select [:source_id :source_db_id :db_name :schema :name- :description :display_name :target_id]- :from [:metabase_table_import]- :order-by [[:source_id :asc]]}))--(defn- staging-fields- "Return the contents of `metabase_field_import` ordered by `source_id`."- []- (t2/query {:select [:source_id :source_table_id :source_parent_id :source_fk_target_id- :name :base_type :database_type :effective_type :semantic_type- :coercion_strategy :description :nfc_path- :depth :target_id :target_table_id :target_parent_id :target_fk_target_id]- :from [:metabase_field_import]- :order-by [[:source_id :asc]]}))--(defn- batch- "Wrap rows in `[line-num row]` tuples as the parser would emit."- [rows]- (mapv (fn [i row] [(inc i) row]) (range) rows))--(defn- drain-tables!- "Open a connection + PreparedStatement and drain `tbl-batch` via the JDBC- variant. For tests that don't care which production path runs."- [databases-by-source-id tbl-batch]- (with-open [^Connection conn (.getConnection (mdb/data-source))- ^PreparedStatement ps (.prepareStatement conn (p/tables-insert-sql))]- (p/drain-tables-batch-jdbc! ps databases-by-source-id tbl-batch)))--(defn- drain-fields!- "Open a connection + PreparedStatement and drain `fld-batch` via the JDBC- variant. For tests that don't care which production path runs."- [fld-batch]- (with-open [^Connection conn (.getConnection (mdb/data-source))- ^PreparedStatement ps (.prepareStatement conn (p/fields-insert-sql))]- (p/drain-fields-batch-jdbc! ps fld-batch)))--;;; ============================== process-databases! ==============================--(deftest process-databases!-matches-by-name-and-engine-test- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}]- (let [results (vec (p/process-databases!- (batch [{:id 100 :name db-name :engine "h2"}])))]- (testing "matched result carries source-id (int), name, target-id"- (is (= [{:source-id 100 :name db-name :target-id db-id :status :matched}]- results))))))--(deftest process-databases!-no-match-test- (let [results (vec (p/process-databases!- (batch [{:id 200 :name "nonexistent-db" :engine "h2"}])))]- (testing "missing target produces :no-match (non-fatal)"- (is (= 1 (count results)))- (is (= :no-match (:status (first results))))- (is (= 200 (:source-id (first results))))- (is (= "nonexistent-db" (:name (first results)))))))--;;; ============================== tables drain ==============================--(deftest drain-tables-writes-source-and-target-columns-test- (try- (p/clear-staging-tables!)- (let [databases-by-source-id {7 "warehouse-a" 9 "warehouse-b"}]- (drain-tables!- databases-by-source-id- (batch [{:id 100 :db_id 7 :schema "public" :name "users"- :description "user table"}- {:id 101 :db_id 7 :schema nil :name "events"}- {:id 200 :db_id 9 :schema "analytics" :name "purchases"}]))- (let [rows (staging-tables)]- (testing "three rows landed"- (is (= 3 (count rows))))- (testing "source ids and natural-key columns populated"- (is (= [100 7 "warehouse-a" "public" "users" "user table"]- ((juxt :source_id :source_db_id :db_name :schema :name :description) (nth rows 0))))- (is (= [101 7 "warehouse-a" nil "events" nil]- ((juxt :source_id :source_db_id :db_name :schema :name :description) (nth rows 1))))- (is (= [200 9 "warehouse-b" "analytics" "purchases" nil]- ((juxt :source_id :source_db_id :db_name :schema :name :description) (nth rows 2)))))- (testing "display_name pre-humanized at drain time"- (is (= "Users" (:display_name (nth rows 0))))- (is (= "Events" (:display_name (nth rows 1))))- (is (= "Purchases" (:display_name (nth rows 2)))))- (testing "target_id starts NULL — populated by the resolve step"- (is (every? nil? (map :target_id rows))))))- (finally (p/clear-staging-tables!))))--;;; ============================== fields drain ==============================--(deftest drain-fields-writes-all-shapes-test- (try- (p/clear-staging-tables!)- (drain-fields!- (batch- ;; Flat field — no parent, no FK, no nfc_path- [{:id 1000 :table_id 100 :name "id"- :base_type "type/Integer" :database_type "int"}- ;; Dictionary parent- {:id 1001 :table_id 100 :name "data"- :base_type "type/Dictionary" :database_type "json"}- ;; nested leaf — parent_id + nfc_path- {:id 1002 :table_id 100 :name "value"- :base_type "type/Text" :database_type "text"- :parent_id 1001- :nfc_path ["data" "value"]}- ;; unfolded leaf — nfc_path only, no parent_id- {:id 1003 :table_id 100 :name "embedded → leaf"- :base_type "type/Text" :database_type "text"- :nfc_path ["embedded" "leaf"]}- ;; FK source- {:id 1004 :table_id 100 :name "account_fk"- :base_type "type/Integer" :database_type "int"- :fk_target_field_id 2000}- ;; Field with effective_type set explicitly (≠ base_type case)- {:id 1005 :table_id 100 :name "casted_value"- :base_type "type/Text" :database_type "varchar"- :effective_type "type/Text"- :semantic_type "type/Email"- :coercion_strategy "Coercion/UNIXSeconds->DateTime"- :description "an email field"}]))- (let [rows (staging-fields)- by-id (into {} (map (juxt :source_id identity) rows))]- (testing "six rows landed"- (is (= 6 (count rows))))- (testing "flat field has no parent or FK source ids"- (let [r (by-id 1000)]- (is (nil? (:source_parent_id r)))- (is (nil? (:source_fk_target_id r)))- (is (nil? (:nfc_path r)))))- (testing "Dictionary parent row has no parent_id and no nfc_path"- (let [r (by-id 1001)]- (is (nil? (:source_parent_id r)))- (is (nil? (:nfc_path r)))))- (testing "nested leaf carries source_parent_id pointing at the parent's source_id"- (let [r (by-id 1002)]- (is (= 1001 (:source_parent_id r)))- (is (= ["data" "value"] (json/decode (:nfc_path r))))))- (testing "unfolded leaf has nfc_path but no source_parent_id"- (let [r (by-id 1003)]- (is (nil? (:source_parent_id r)))- (is (= ["embedded" "leaf"] (json/decode (:nfc_path r))))))- (testing "FK source carries source_fk_target_id"- (let [r (by-id 1004)]- (is (= 2000 (:source_fk_target_id r)))))- (testing "optional payload columns roundtrip through staging"- (let [r (by-id 1005)]- (is (= "type/Text" (:effective_type r)))- (is (= "type/Email" (:semantic_type r)))- (is (= "Coercion/UNIXSeconds->DateTime" (:coercion_strategy r)))- (is (= "an email field" (:description r)))))- (testing "resolve outputs all start NULL"- (doseq [{:keys [depth target_id target_table_id target_parent_id target_fk_target_id]} rows]- (is (nil? depth))- (is (nil? target_id))- (is (nil? target_table_id))- (is (nil? target_parent_id))- (is (nil? target_fk_target_id)))))- (finally (p/clear-staging-tables!))))--;;; ============================== empty-batch behavior ==============================--(deftest drain-tables-empty-batch-is-noop-test- (try- (p/clear-staging-tables!)- (drain-tables! {} (batch []))- (is (zero? (count (staging-tables))))- (finally (p/clear-staging-tables!))))--(deftest drain-fields-empty-batch-is-noop-test- (try- (p/clear-staging-tables!)- (drain-fields! (batch []))- (is (zero? (count (staging-fields))))- (finally (p/clear-staging-tables!))))--;;; ============================== validation failures ==============================--(deftest drain-tables-rejects-rows-missing-required-keys-test- (try- (p/clear-staging-tables!)- (testing "missing :id throws with :kind :invalid-input"- (is (= :invalid-input- (try- (drain-tables! {} (batch [{:db_id 7 :name "no-id" :schema "public"}]))- nil- (catch clojure.lang.ExceptionInfo e (:kind (ex-data e)))))))- (testing "no rows landed in staging on validation failure"- (is (zero? (count (staging-tables)))))- (finally (p/clear-staging-tables!))))--(deftest drain-fields-rejects-rows-missing-required-keys-test- (try- (p/clear-staging-tables!)- (testing "missing :base_type throws with :kind :invalid-input"- (is (= :invalid-input- (try- (drain-fields! (batch [{:id 1 :table_id 100 :name "x" :database_type "text"}]))- nil- (catch clojure.lang.ExceptionInfo e (:kind (ex-data e)))))))- (finally (p/clear-staging-tables!))))--;;; ============================== Drain-path equivalence ==============================--(def ^:private equivalence-databases-by-source-id- {7 "warehouse-a" 9 "warehouse-b"})--(def ^:private equivalence-table-batch-rows- [{:id 100 :db_id 7 :schema "public" :name "users" :description "user table"}- {:id 101 :db_id 7 :schema nil :name "events"}- {:id 200 :db_id 9 :schema "analytics" :name "purchases"}])--(def ^:private equivalence-field-batch-rows- [{:id 1000 :table_id 100 :name "id"- :base_type "type/Integer" :database_type "int"}- {:id 1001 :table_id 100 :name "data"- :base_type "type/Dictionary" :database_type "json"}- {:id 1002 :table_id 100 :name "value"- :base_type "type/Text" :database_type "text"- :parent_id 1001 :nfc_path ["data" "value"]}- {:id 1003 :table_id 100 :name "embedded → leaf"- :base_type "type/Text" :database_type "text"- :nfc_path ["embedded" "leaf"]}- {:id 1004 :table_id 100 :name "account_fk"- :base_type "type/Integer" :database_type "int"- :fk_target_field_id 2000}- {:id 1005 :table_id 100 :name "casted_value"- :base_type "type/Text" :database_type "varchar"- :effective_type "type/Text" :semantic_type "type/Email"- :coercion_strategy "Coercion/UNIXSeconds->DateTime"- :description "an email field"}])--(defn- snapshot-after! [drain-fn]- (try- (p/clear-staging-tables!)- (drain-fn)- {:tables (staging-tables) :fields (staging-fields)}- (finally (p/clear-staging-tables!))))--(deftest drain-paths-produce-identical-staging-test- (testing "the JDBC and PG-COPY drain paths produce the same staging-table- contents for the same batch — guards against silent drift if- one path's row-shaping diverges from the other. Only meaningful- on a PG appdb (the JDBC variant runs on every appdb; the- PG-COPY variant runs only on PG)."- (when (= :postgres (mdb/db-type))- (let [tbl-batch (batch equivalence-table-batch-rows)- fld-batch (batch equivalence-field-batch-rows)- jdbc (snapshot-after!- (fn []- (with-open [^Connection conn (.getConnection (mdb/data-source))- ^PreparedStatement tps (.prepareStatement conn (p/tables-insert-sql))- ^PreparedStatement fps (.prepareStatement conn (p/fields-insert-sql))]- (p/drain-tables-batch-jdbc! tps equivalence-databases-by-source-id tbl-batch)- (p/drain-fields-batch-jdbc! fps fld-batch))))- pg-copy (snapshot-after!- (fn []- (with-open [^Connection conn (.getConnection (mdb/data-source))]- (let [^PGConnection pg-conn (.unwrap conn PGConnection)- copy-mgr (.getCopyAPI pg-conn)- t-ci (.copyIn copy-mgr ^String p/tables-copy-sql)]- (try- (p/drain-tables-batch-pg-copy! t-ci equivalence-databases-by-source-id tbl-batch)- (finally (.endCopy ^CopyIn t-ci))))- (let [^PGConnection pg-conn (.unwrap conn PGConnection)- copy-mgr (.getCopyAPI pg-conn)- f-ci (.copyIn copy-mgr ^String p/fields-copy-sql)]- (try- (p/drain-fields-batch-pg-copy! f-ci fld-batch)- (finally (.endCopy ^CopyIn f-ci)))))))]- (is (= jdbc pg-copy)- "JDBC drain produces the same staging rows as the PG-COPY drain")))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/fields_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/fields_test.cljdeleted file mode 100644index cde6b147089..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/fields_test.clj+++ /dev/null@@ -1,497 +0,0 @@-(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.fields-test- "Tests for the per-depth field-merge functions: the three resolves- (`resolve-target-table-ids-for-fields-in-staging!`,- `fill-target-parent-ids-at-depth!`, `fill-target-fk-target-ids-at-depth!`,- `resolve-target-field-ids-at-depth!`), the UPDATE/INSERT pair- (`update-matched-fields-at-depth!`, `insert-new-fields-at-depth!`), and- the orchestration loop (`merge-fields-by-depth!`)."- (:require- [clojure.string :as str]- [clojure.test :refer :all]- [metabase-enterprise.serialization.metadata-file-import.processors :as p]- [metabase.test :as mt]- [metabase.util.json :as json]- [toucan2.core :as t2]))--(use-fixtures :once- (fn [thunk]- ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with- ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.- (mt/db)- (mt/with-temporary-setting-values [disable-auto-sync true]- (thunk))))--(defn- insert-staging-table-row!- "Helper: drop one row into metabase_table_import. `overrides` can include- `:target_id` to pre-resolve the row."- [source-id db-name name & {:as overrides}]- (t2/insert! :metabase_table_import- (merge {:source_id source-id- :source_db_id 1- :db_name db-name- :schema "PUBLIC"- :name name- :display_name (str/capitalize name)}- overrides)))--(defn- insert-staging-field-row!- "Helper: drop one row into metabase_field_import. Fills in NOT NULL- defaults; `overrides` is everything else."- [source-id name & {:as overrides}]- (t2/insert! :metabase_field_import- (merge {:source_id source-id- :source_table_id 1- :name name- :base_type "type/Integer"- :database_type "int"}- overrides)))--(defn- field-staging-row- "Read a staging row by source_id for assertion."- [source-id]- (t2/select-one- [:metabase_field_import- :source_id :source_table_id :source_parent_id :source_fk_target_id- :name :base_type :database_type :nfc_path :depth- :target_id :target_table_id :target_parent_id :target_fk_target_id]- :source_id source-id))--;;; ============================== resolve-target-table-ids-for-fields ==============================--(deftest resolve-target-table-ids-for-fields-copies-from-table-staging-test- (try- (p/clear-staging-tables!)- ;; Two staging tables, one pre-resolved (target_id = 42), one not.- (insert-staging-table-row! 1 "warehouse" "users" :target_id 42)- (insert-staging-table-row! 2 "warehouse" "ghost")- (insert-staging-field-row! 100 "id" :source_table_id 1)- (insert-staging-field-row! 200 "x" :source_table_id 2)- (p/resolve-target-table-ids-for-fields-in-staging!)- (is (= 42 (:target_table_id (field-staging-row 100))))- (is (nil? (:target_table_id (field-staging-row 200)))- "field whose table has no target_id stays NULL")- (finally (p/clear-staging-tables!))))--;;; ============================== fill-target-parent-ids-at-depth ==============================--(deftest fill-target-parent-ids-only-touches-the-given-depth-test- (try- (p/clear-staging-tables!)- ;; root at depth 0 (target_id = 1000 — pretend it's already been merged)- (insert-staging-field-row! 10 "root" :depth 0 :target_id 1000)- ;; child at depth 1 with parent pointing at the root- (insert-staging-field-row! 11 "child" :depth 1 :source_parent_id 10)- ;; another row at depth 2 — should NOT be touched by depth=1 call- (insert-staging-field-row! 12 "grandchild" :depth 2 :source_parent_id 11)- (p/fill-target-parent-ids-at-depth! 1)- (is (= 1000 (:target_parent_id (field-staging-row 11)))- "depth-1 row's target_parent_id picked up from parent at depth 0")- (is (nil? (:target_parent_id (field-staging-row 12)))- "depth-2 row not touched by depth=1 call")- (finally (p/clear-staging-tables!))))--(deftest fill-target-parent-ids-skips-rows-without-source-parent-test- (try- (p/clear-staging-tables!)- (insert-staging-field-row! 10 "root" :depth 0 :target_id 1000)- (insert-staging-field-row! 11 "no-parent" :depth 1)- (p/fill-target-parent-ids-at-depth! 1)- (is (nil? (:target_parent_id (field-staging-row 11))))- (finally (p/clear-staging-tables!))))--;;; ============================== fill-target-fk-target-ids-at-depth ==============================--(deftest fill-target-fk-target-ids-at-depth-test- (try- (p/clear-staging-tables!)- (insert-staging-field-row! 50 "id" :depth 0 :target_id 5000)- (insert-staging-field-row! 51 "account_fk" :depth 1 :source_fk_target_id 50)- (p/fill-target-fk-target-ids-at-depth! 1)- (is (= 5000 (:target_fk_target_id (field-staging-row 51))))- (finally (p/clear-staging-tables!))))--;;; ============================== resolve-target-field-ids-at-depth ==============================--(deftest resolve-target-field-ids-matches-by-natural-key-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}- :model/Field {target-id :id} {:table_id t-id :name "user_id"- :base_type :type/Integer}]- (try- (p/clear-staging-tables!)- (insert-staging-field-row! 100 "user_id" :depth 0 :target_table_id t-id)- (p/resolve-target-field-ids-at-depth! 0)- (is (= target-id (:target_id (field-staging-row 100))))- (finally (p/clear-staging-tables!)))))--(deftest resolve-target-field-ids-honors-parent-id-in-match-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}- :model/Field {parent-id :id} {:table_id t-id :name "data" :base_type :type/Dictionary}- :model/Field {child-id :id} {:table_id t-id :name "value"- :base_type :type/Text :parent_id parent-id}- ;; Sibling at the root level with the same name as the nested child- :model/Field {root-id :id} {:table_id t-id :name "value" :base_type :type/Text}]- (try- (p/clear-staging-tables!)- (insert-staging-field-row! 100 "value" :depth 1 :target_table_id t-id- :target_parent_id parent-id)- (insert-staging-field-row! 101 "value" :depth 0 :target_table_id t-id)- (p/resolve-target-field-ids-at-depth! 1)- (p/resolve-target-field-ids-at-depth! 0)- (is (= child-id (:target_id (field-staging-row 100)))- "nested 'value' (parent_id set) matches the nested target")- (is (= root-id (:target_id (field-staging-row 101)))- "flat 'value' (parent_id NULL) matches the root target")- (finally (p/clear-staging-tables!)))))--(deftest resolve-target-field-ids-leaves-null-when-no-match-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]- (try- (p/clear-staging-tables!)- (insert-staging-field-row! 100 "no_such_field" :depth 0 :target_table_id t-id)- (p/resolve-target-field-ids-at-depth! 0)- (is (nil? (:target_id (field-staging-row 100))))- (finally (p/clear-staging-tables!)))))--(deftest resolve-target-field-ids-skips-defective-duplicate-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}- :model/Field {} {:table_id t-id :name "user_id"- :base_type :type/Integer- :is_defective_duplicate true}]- (try- (p/clear-staging-tables!)- (insert-staging-field-row! 100 "user_id" :depth 0 :target_table_id t-id)- (p/resolve-target-field-ids-at-depth! 0)- (is (nil? (:target_id (field-staging-row 100))))- (finally (p/clear-staging-tables!)))))--(deftest resolve-target-field-ids-is-idempotent-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}- :model/Field {target-id :id} {:table_id t-id :name "x" :base_type :type/Text}]- (try- (p/clear-staging-tables!)- (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id)- (p/resolve-target-field-ids-at-depth! 0)- (p/resolve-target-field-ids-at-depth! 0)- (is (= target-id (:target_id (field-staging-row 100))))- (finally (p/clear-staging-tables!)))))--;;; ============================== update-matched-fields-at-depth ==============================--(deftest update-matched-clobbers-payload-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}- :model/Field {target-id :id} {:table_id t-id :name "x"- :base_type :type/Integer- :description "old"- :database_type "int"- :semantic_type nil}]- (try- (p/clear-staging-tables!)- (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id :target_id target-id- :base_type "type/Integer" :database_type "bigint"- :description "new" :semantic_type "type/PK")- (p/update-matched-fields-at-depth! 0)- (let [after (t2/select-one :model/Field :id target-id)]- (is (= "new" (:description after)))- (is (= "bigint" (:database_type after)))- (is (= :type/PK (:semantic_type after))))- (finally (p/clear-staging-tables!)))))--(deftest update-matched-skip-if-unchanged-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}- :model/Field {target-id :id} {:table_id t-id :name "x"- :base_type :type/Integer- :description "same"- :database_type "int"- :active true}]- (try- (t2/query {:update :metabase_field- :set {:updated_at #t "2020-01-01T00:00:00Z"}- :where [:= :id target-id]})- (let [orig (:updated_at (t2/select-one :model/Field :id target-id))]- (p/clear-staging-tables!)- ;; Staging carries identical payload — UPDATE must NOT fire.- (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id :target_id target-id- :base_type "type/Integer" :database_type "int"- :description "same")- (p/update-matched-fields-at-depth! 0)- (is (= orig (:updated_at (t2/select-one :model/Field :id target-id)))- "identical payload → no UPDATE → updated_at unchanged"))- (finally (p/clear-staging-tables!)))))--(deftest update-matched-fires-when-payload-differs-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}- :model/Field {target-id :id} {:table_id t-id :name "x"- :base_type :type/Integer- :description "old"- :database_type "int"}]- (try- (t2/query {:update :metabase_field- :set {:updated_at #t "2020-01-01T00:00:00Z"}- :where [:= :id target-id]})- (let [orig (:updated_at (t2/select-one :model/Field :id target-id))]- (p/clear-staging-tables!)- (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id :target_id target-id- :base_type "type/Integer" :database_type "int"- :description "new")- (p/update-matched-fields-at-depth! 0)- (is (not= orig (:updated_at (t2/select-one :model/Field :id target-id)))- "different payload → UPDATE fires → updated_at bumped"))- (finally (p/clear-staging-tables!)))))--;;; ============================== insert-new-fields-at-depth ==============================--(deftest insert-new-inserts-root-field-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]- (try- (p/clear-staging-tables!)- (insert-staging-field-row! 100 "user_id" :depth 0 :target_table_id t-id- :base_type "type/Integer" :database_type "int")- (p/insert-new-fields-at-depth! 0)- (let [inserted (t2/select-one :model/Field :table_id t-id :name "user_id")]- (is (some? inserted))- (is (= :type/Integer (:base_type inserted)))- (is (= "int" (:database_type inserted)))- (is (true? (:active inserted)))- (is (nil? (:parent_id inserted)))- (is (nil? (:fk_target_field_id inserted))))- (finally (p/clear-staging-tables!)))))--(deftest insert-new-uses-target-parent-id-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}- :model/Field {parent-id :id} {:table_id t-id :name "data" :base_type :type/Dictionary}]- (try- (p/clear-staging-tables!)- (insert-staging-field-row! 100 "value" :depth 1 :target_table_id t-id- :base_type "type/Text" :database_type "text"- :target_parent_id parent-id- :nfc_path (json/encode ["data" "value"]))- (p/insert-new-fields-at-depth! 1)- (let [inserted (t2/select-one :model/Field :table_id t-id :name "value" :parent_id parent-id)]- (is (some? inserted))- (is (= parent-id (:parent_id inserted)))- (is (= ["data" "value"] (:nfc_path inserted))))- (finally (p/clear-staging-tables!)))))--(deftest insert-new-uses-target-fk-target-id-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}- :model/Field {target-id :id} {:table_id t-id :name "id" :base_type :type/Integer}]- (try- (p/clear-staging-tables!)- (insert-staging-field-row! 100 "account_fk" :depth 1 :target_table_id t-id- :base_type "type/Integer" :database_type "int"- :target_fk_target_id target-id)- (p/insert-new-fields-at-depth! 1)- (let [inserted (t2/select-one :model/Field :table_id t-id :name "account_fk")]- (is (= target-id (:fk_target_field_id inserted))))- (finally (p/clear-staging-tables!)))))--(deftest insert-new-preserves-null-effective-type-test- ;; The application reads NULL effective_type as "equal to base_type" at- ;; use sites, so the stored NULL is correct.- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]- (try- (p/clear-staging-tables!)- (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id- :base_type "type/Integer" :database_type "int"- :effective_type nil)- (p/insert-new-fields-at-depth! 0)- (let [inserted (t2/select-one :model/Field :table_id t-id :name "x")]- (is (some? inserted))- (is (nil? (:effective_type inserted))- "NULL staging effective_type → NULL target effective_type"))- (finally (p/clear-staging-tables!)))))--(deftest insert-new-preserves-explicit-effective-type-test- ;; When source had effective_type ≠ base_type, the wire row carries the- ;; explicit value. Staging carries it. INSERT writes it.- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]- (try- (p/clear-staging-tables!)- (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id- :base_type "type/Text" :database_type "varchar"- :effective_type "type/PostgresEnum")- (p/insert-new-fields-at-depth! 0)- (let [inserted (t2/select-one :model/Field :table_id t-id :name "x")]- (is (= :type/PostgresEnum (:effective_type inserted))))- (finally (p/clear-staging-tables!)))))--(deftest update-matched-skip-if-unchanged-handles-effective-type-coalesce-test- ;; Cross-checks the COALESCE branch in field-payload-changed-predicate:- ;; target.effective_type explicitly == base_type, staging.effective_type- ;; NULL → COALESCED values match → skip-if-unchanged fires.- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}- :model/Field {target-id :id} {:table_id t-id :name "x"- :base_type :type/Integer- :effective_type :type/Integer- :database_type "int"}]- (try- (t2/query {:update :metabase_field- :set {:updated_at #t "2020-01-01T00:00:00Z"}- :where [:= :id target-id]})- (let [orig (:updated_at (t2/select-one :model/Field :id target-id))]- (p/clear-staging-tables!)- ;; Staging matches except effective_type is NULL (the export-omitted case).- (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id :target_id target-id- :base_type "type/Integer" :database_type "int"- :effective_type nil)- (p/update-matched-fields-at-depth! 0)- (is (= orig (:updated_at (t2/select-one :model/Field :id target-id)))- "COALESCE(target.effective_type, target.base_type) == COALESCE(NULL, staging.base_type) — no UPDATE"))- (finally (p/clear-staging-tables!)))))--(deftest insert-new-skips-row-with-no-target-table-test- (try- (p/clear-staging-tables!)- (let [count-before (t2/count :metabase_field)]- (insert-staging-field-row! 100 "orphan" :depth 0 :target_table_id nil)- (p/insert-new-fields-at-depth! 0)- (is (= count-before (t2/count :metabase_field))))- (finally (p/clear-staging-tables!))))--(deftest insert-new-skips-row-whose-parent-was-orphaned-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]- (try- (p/clear-staging-tables!)- ;; Row has a source_parent_id but target_parent_id wasn't resolved- ;; (would happen if the parent's table didn't match target).- (insert-staging-field-row! 100 "orphaned_child" :depth 1- :target_table_id t-id- :source_parent_id 999)- (p/insert-new-fields-at-depth! 1)- (is (zero? (t2/count :model/Field :table_id t-id :name "orphaned_child")))- (finally (p/clear-staging-tables!)))))--(deftest insert-new-keeps-row-whose-fk-target-was-dropped-test- (testing "a field whose FK target was dropped during merge (source_fk_target_id- set, target_fk_target_id unresolved) is still inserted with- fk_target_field_id NULL — the FK is a semantic annotation, not structural."- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]- (try- (p/clear-staging-tables!)- ;; Field sits in a matched table (target_table_id set) and has an FK whose- ;; target was dropped: source_fk_target_id is set, target_fk_target_id NULL.- (insert-staging-field-row! 100 "user_id" :depth 1- :target_table_id t-id- :source_fk_target_id 999)- (p/insert-new-fields-at-depth! 1)- (let [inserted (t2/select-one :model/Field :table_id t-id :name "user_id")]- (is (some? inserted)- "field in a matched table is inserted even though its FK target didn't resolve")- (is (nil? (:fk_target_field_id inserted))- "the unresolved FK is stored as NULL, not the field dropped"))- (finally (p/clear-staging-tables!))))))--;;; ============================== merge-fields-by-depth (integration) ==============================--(deftest merge-fields-by-depth-handles-flat-fields-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]- (try- (p/clear-staging-tables!)- (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)- (insert-staging-field-row! 100 "a" :depth 0 :target_table_id t-id)- (insert-staging-field-row! 101 "b" :depth 0 :target_table_id t-id)- (p/merge-fields-by-depth!)- (is (= #{"a" "b"} (set (map :name (t2/select :model/Field :table_id t-id)))))- (finally (p/clear-staging-tables!)))))--(deftest merge-fields-by-depth-handles-parent-chain-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]- (try- (p/clear-staging-tables!)- (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)- ;; root → parent → grandchild- (insert-staging-field-row! 10 "data" :depth 0 :target_table_id t-id- :base_type "type/Dictionary" :database_type "json")- (insert-staging-field-row! 11 "inner" :depth 1 :target_table_id t-id- :base_type "type/Dictionary" :database_type "json"- :source_parent_id 10- :nfc_path (json/encode ["data"]))- (insert-staging-field-row! 12 "leaf" :depth 2 :target_table_id t-id- :base_type "type/Text" :database_type "text"- :source_parent_id 11- :nfc_path (json/encode ["data" "inner"]))- (p/merge-fields-by-depth!)- (let [root (t2/select-one :model/Field :table_id t-id :name "data")- inner (t2/select-one :model/Field :table_id t-id :name "inner")- leaf (t2/select-one :model/Field :table_id t-id :name "leaf")]- (is (some? root))- (is (= (:id root) (:parent_id inner)))- (is (= (:id inner) (:parent_id leaf)))- (is (= ["data" "inner"] (:nfc_path leaf))))- (finally (p/clear-staging-tables!)))))--(deftest merge-fields-by-depth-handles-fk-chain-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]- (try- (p/clear-staging-tables!)- (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)- ;; A is non-FK, B points at A via fk_target.- (insert-staging-field-row! 10 "a" :depth 0 :target_table_id t-id)- (insert-staging-field-row! 11 "b" :depth 1 :target_table_id t-id- :source_fk_target_id 10)- (p/merge-fields-by-depth!)- (let [a (t2/select-one :model/Field :table_id t-id :name "a")- b (t2/select-one :model/Field :table_id t-id :name "b")]- (is (some? a))- (is (some? b))- (is (= (:id a) (:fk_target_field_id b))))- (finally (p/clear-staging-tables!)))))--(deftest merge-fields-by-depth-reconciles-fk-on-matched-field-test- (testing "a matched field's fk_target_field_id is reconciled from the import- file, even when no other payload column differs — the FK isn't part- of the natural-key match, so it can drift independently of row identity."- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}- :model/Field {id-field :id} {:table_id t-id :name "id"- :base_type :type/Integer :database_type "int"}- :model/Field {acct-field :id} {:table_id t-id :name "account_id"- :base_type :type/Integer :database_type "int"}]- (try- (is (nil? (:fk_target_field_id (t2/select-one :model/Field :id acct-field)))- "fixture starts with account_id un-FK'd")- (p/clear-staging-tables!)- (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)- ;; Staging payloads are identical to the live rows except account_id now- ;; carries an FK pointing at id — so only the FK has drifted.- (insert-staging-field-row! 10 "id" :depth 0 :target_table_id t-id)- (insert-staging-field-row! 11 "account_id" :depth 1 :target_table_id t-id- :source_fk_target_id 10)- (p/merge-fields-by-depth!)- (is (= id-field (:fk_target_field_id (t2/select-one :model/Field :id acct-field)))- "matched field's fk_target_field_id is reconciled from the file")- (finally (p/clear-staging-tables!))))))--(deftest merge-fields-by-depth-is-idempotent-test- (mt/with-temp [:model/Database {db-id :id} {:engine :h2}- :model/Table {t-id :id} {:db_id db-id :schema "PUBLIC" :name "t"}]- (try- (p/clear-staging-tables!)- (insert-staging-table-row! 1 (:name (t2/select-one :model/Database :id db-id)) "t" :target_id t-id)- (insert-staging-field-row! 100 "x" :depth 0 :target_table_id t-id)- (p/merge-fields-by-depth!)- (let [first-id (:id (t2/select-one :model/Field :table_id t-id :name "x"))- _ (p/merge-fields-by-depth!)- second-id (:id (t2/select-one :model/Field :table_id t-id :name "x"))]- (is (= first-id second-id))- (is (= 1 (t2/count :model/Field :table_id t-id :name "x"))))- (finally (p/clear-staging-tables!)))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/orphan_check_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/orphan_check_test.cljdeleted file mode 100644index 45bf495c0ed..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/orphan_check_test.clj+++ /dev/null@@ -1,167 +0,0 @@-(ns metabase-enterprise.serialization.metadata-file-import.orphan-check-test- "Tests for the pre-flight orphan handling that runs after drain and before- the depth-walk merge. Asymmetric by ref kind:-- - `source_parent_id` orphans → fatal (`assert-no-orphan-refs!` throws- `:file-incomplete`). A field claiming a non-present parent has no- structurally valid position.- - `source_fk_target_id` orphans → lossy (`null-orphan-fk-target-refs!`- NULLs the ref and returns the count for the caller to WARN-log).- A missing fk target degrades cleanly to 'no fk relationship known'."- (:require- [clojure.test :refer :all]- [metabase-enterprise.serialization.metadata-file-import.processors :as p]- [toucan2.core :as t2]))--(defn- insert-staging-fields!- "Insert a sequence of partial field rows directly into `metabase_field_import`,- filling in required columns with sensible defaults so we can focus the test- on the source_parent_id / source_fk_target_id graph."- [rows]- (let [defaults {:name "x" :base_type "type/Integer" :database_type "int"}]- (t2/insert! :metabase_field_import- (mapv (fn [row] (merge defaults {:source_table_id 1} row)) rows))))--;;; ============================== happy path ==============================--(deftest empty-staging-passes-test- (try- (p/clear-staging-tables!)- (is (nil? (p/assert-no-orphan-refs!)))- (finally (p/clear-staging-tables!))))--(deftest staging-with-only-roots-passes-test- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 1}- {:source_id 2}- {:source_id 3}])- (is (nil? (p/assert-no-orphan-refs!)))- (finally (p/clear-staging-tables!))))--(deftest staging-with-resolvable-parent-and-fk-passes-test- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 10}- {:source_id 11 :source_parent_id 10}- {:source_id 12 :source_fk_target_id 11}- {:source_id 13 :source_parent_id 11 :source_fk_target_id 10}])- (is (nil? (p/assert-no-orphan-refs!)))- (finally (p/clear-staging-tables!))))--;;; ============================== orphan parent ref ==============================--(deftest single-orphan-parent-ref-throws-test- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 100}- {:source_id 101 :source_parent_id 999}])- (let [thrown (try (p/assert-no-orphan-refs!) nil- (catch clojure.lang.ExceptionInfo e e))]- (is (some? thrown) "should have thrown")- (is (= :file-incomplete (:kind (ex-data thrown))))- (is (= 1 (:orphan-parent-count (ex-data thrown))))- (is (= [{:source_id 101 :source_parent_id 999}]- (:orphan-parent-sample (ex-data thrown))))- (testing "error message includes the count"- (is (re-find #"1 orphan parent" (ex-message thrown)))))- (finally (p/clear-staging-tables!))))--;;; ============================== orphan fk-target ref ==============================--(deftest orphan-fk-target-ref-is-not-fatal-for-assert-test- ;; Handled separately by null-orphan-fk-target-refs!.- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 200}- {:source_id 201 :source_fk_target_id 9999}])- (is (nil? (p/assert-no-orphan-refs!))- "orphan fk-target alone should not throw")- (finally (p/clear-staging-tables!))))--(deftest orphan-fk-target-ref-gets-nulled-test- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 200}- {:source_id 201 :source_fk_target_id 9999}])- (let [result (p/null-orphan-fk-target-refs!)]- (is (= 1 (:count result)))- (is (= [{:source_id 201 :source_fk_target_id 9999}]- (:sample result))))- (testing "the orphan ref is NULL'd in staging"- (is (= [{:source_id 201 :source_fk_target_id nil}]- (t2/query- {:select [:source_id :source_fk_target_id]- :from [:metabase_field_import]- :where [:= :source_id 201]})))- (testing "the non-orphan row is untouched"- (is (= [{:source_id 200 :source_fk_target_id nil}]- (t2/query- {:select [:source_id :source_fk_target_id]- :from [:metabase_field_import]- :where [:= :source_id 200]})))))- (finally (p/clear-staging-tables!))))--(deftest valid-fk-target-ref-survives-null-pass-test- ;; null-orphan-fk-target-refs! must not touch rows whose fk target IS- ;; present in staging.- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 210}- {:source_id 211 :source_fk_target_id 210}])- (let [result (p/null-orphan-fk-target-refs!)]- (is (= 0 (:count result)))- (is (nil? (:sample result))))- (is (= [{:source_id 211 :source_fk_target_id 210}]- (t2/query- {:select [:source_id :source_fk_target_id]- :from [:metabase_field_import]- :where [:= :source_id 211]})))- (finally (p/clear-staging-tables!))))--;;; ============================== both kinds of orphan ==============================--(deftest both-orphan-types-handled-asymmetrically-test- ;; Orphan parent still throws; orphan fk-target is handled by the separate- ;; null-pass. The throw happens before the null-pass would run.- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 300}- {:source_id 301 :source_parent_id 9001}- {:source_id 302 :source_fk_target_id 9002}])- (let [thrown (try (p/assert-no-orphan-refs!) nil- (catch clojure.lang.ExceptionInfo e e))]- (is (some? thrown) "assert-no-orphan-refs! should still throw on parent orphan")- (is (= :file-incomplete (:kind (ex-data thrown))))- (is (= 1 (:orphan-parent-count (ex-data thrown))))- (is (seq (:orphan-parent-sample (ex-data thrown)))))- (finally (p/clear-staging-tables!))))--;;; ============================== sample truncation ==============================--(deftest orphan-sample-caps-at-ten-rows-test- (try- (p/clear-staging-tables!)- ;; 25 rows all pointing at a non-existent parent- (insert-staging-fields!- (cons {:source_id 1}- (for [i (range 1000 1025)] {:source_id i :source_parent_id 999})))- (let [thrown (try (p/assert-no-orphan-refs!) nil- (catch clojure.lang.ExceptionInfo e e))]- (testing "count reflects all 25 orphans"- (is (= 25 (:orphan-parent-count (ex-data thrown)))))- (testing "sample is capped at 10"- (is (= 10 (count (:orphan-parent-sample (ex-data thrown)))))))- (finally (p/clear-staging-tables!))))--;;; ============================== self-reference is not an orphan ==============================--(deftest self-referential-parent-id-is-not-an-orphan-test- ;; A row whose source_parent_id equals its own source_id is a cycle —- ;; that's a different problem (caught by depth tagging) but it's NOT an- ;; orphan since the referenced source_id IS in the file.- (try- (p/clear-staging-tables!)- (insert-staging-fields! [{:source_id 500 :source_parent_id 500}])- (is (nil? (p/assert-no-orphan-refs!)))- (finally (p/clear-staging-tables!))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/parsers_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/parsers_test.cljdeleted file mode 100644index 4fc46cb6149..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/parsers_test.clj+++ /dev/null@@ -1,125 +0,0 @@-(ns ^:parallel metabase-enterprise.serialization.metadata-file-import.parsers-test- "Tests for the streaming JSON parser. Each test writes a temp file, then- asserts on the batches passed to `process-batch!`."- (:require- [clojure.string :as str]- [clojure.test :refer :all]- [metabase-enterprise.serialization.metadata-file-import.parsers :as parsers])- (:import- (java.io File)))--(set! *warn-on-reflection* true)--(defn- temp-json-file ^File [^String content]- (let [f (File/createTempFile "parsers-test-" ".json")]- (.deleteOnExit f)- (spit f content)- f))--(defn- collect-batches!- "Stream `json-string` through the parser and return a vector of all batches that- the callback received, in order. Each batch is a vector of `[line-num row]` tuples."- [json-string array-key batch-size]- (let [file (temp-json-file json-string)- batches (atom [])]- (try- (parsers/stream-array-batches! file array-key batch-size- (fn [batch] (swap! batches conj (vec batch))))- @batches- (finally (.delete file)))))--(deftest streams-rows-as-keyword-keyed-clojure-maps-test- (testing "each item in the named array is parsed into a Clojure map with keyword keys- (not LinkedHashMap with string keys), wrapped in a [line-num row] tuple"- (let [batches (collect-batches!- "{\"databases\":[{\"id\":17,\"name\":\"pg\",\"engine\":\"postgres\"},- {\"id\":18,\"name\":\"h2\",\"engine\":\"h2\"}]}"- :databases 100)]- (is (= 1 (count batches)))- (is (= [[1 {:id 17 :name "pg" :engine "postgres"}]- [2 {:id 18 :name "h2" :engine "h2"}]]- (first batches))))))--(deftest line-numbers-are-1-indexed-across-batches-test- (testing "line-num is 1-indexed and continues across batch boundaries — so the third- item in the source array is `line-num 3`, regardless of which batch it lands in"- (let [json (str "{\"xs\":["- (str/join "," (for [i (range 1 6)] (format "{\"v\":%d}" i)))- "]}")- batches (collect-batches! json :xs 2)]- (is (= 3 (count batches))) ;; 2 + 2 + 1- (is (= [[1 {:v 1}] [2 {:v 2}]] (nth batches 0)))- (is (= [[3 {:v 3}] [4 {:v 4}]] (nth batches 1)))- (is (= [[5 {:v 5}]] (nth batches 2))))))--(deftest empty-array-invokes-callback-zero-times-test- (testing "if the named array is empty, the callback is not invoked at all — caller- sees no batches, but the parser does not throw"- (is (= [] (collect-batches! "{\"xs\":[]}" :xs 100)))))--(deftest accepts-keyword-or-string-array-key-test- (testing "array-key may be passed as a Clojure keyword or as a string; both forms- normalize to the same JSON key lookup"- (let [json "{\"databases\":[{\"id\":1}]}"]- (is (= [[[1 {:id 1}]]] (collect-batches! json :databases 10)))- (is (= [[[1 {:id 1}]]] (collect-batches! json "databases" 10))))))--(deftest missing-array-key-throws-shape-error-test- (testing "a top-level object that doesn't contain the requested key throws ex-info with- :kind :missing-key"- (try- (collect-batches! "{\"other\":[]}" :databases 10)- (is false "should have thrown")- (catch clojure.lang.ExceptionInfo e- (is (= :missing-key (:kind (ex-data e))))- (is (= "databases" (:key (ex-data e))))))))--(deftest non-array-value-at-key-throws-shape-error-test- (testing "a top-level key whose value is not an array throws ex-info with :kind :bad-shape"- (try- (collect-batches! "{\"databases\":{\"not\":\"an array\"}}" :databases 10)- (is false "should have thrown")- (catch clojure.lang.ExceptionInfo e- (is (= :bad-shape (:kind (ex-data e))))))))--(deftest non-object-top-level-throws-shape-error-test- (testing "a document that doesn't begin with a top-level object throws"- (try- (collect-batches! "[1, 2, 3]" :anything 10)- (is false "should have thrown")- (catch clojure.lang.ExceptionInfo e- (is (= :bad-shape (:kind (ex-data e))))))))--(deftest skips-other-top-level-keys-before-target-test- (testing "the parser advances past unrelated top-level keys (and their values, including- nested structures) before reaching the target array"- (let [json (str "{\"version\":1,"- "\"unrelated_object\":{\"a\":1,\"b\":[2,3]},"- "\"unrelated_array\":[4,5,6],"- "\"databases\":[{\"id\":99}]}")- batches (collect-batches! json :databases 10)]- (is (= [[[1 {:id 99}]]] batches)))))--(deftest exact-batch-size-emits-single-batch-test- (testing "boundary: when item count exactly equals batch size, callback fires once"- (let [json "{\"xs\":[{\"v\":1},{\"v\":2}]}"- batches (collect-batches! json :xs 2)]- (is (= 1 (count batches)))- (is (= 2 (count (first batches)))))))--(deftest single-item-with-batch-size-1-test- (testing "boundary: batch size 1 fires the callback once per item"- (let [json "{\"xs\":[{\"v\":1},{\"v\":2},{\"v\":3}]}"- batches (collect-batches! json :xs 1)]- (is (= 3 (count batches)))- (is (= [[1 {:v 1}]] (nth batches 0)))- (is (= [[2 {:v 2}]] (nth batches 1)))- (is (= [[3 {:v 3}]] (nth batches 2))))))--(deftest file-handle-is-closed-after-streaming-test- (testing "the parser manages the file lifecycle: the InputStream and Reader are- closed after streaming completes (or throws). On Windows a sharing- violation would surface if the FD were leaked."- (let [file (temp-json-file "{\"xs\":[]}")]- (parsers/stream-array-batches! file :xs 10 (fn [_]))- (is (.delete file)))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/processors_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/processors_test.cljdeleted file mode 100644index 2e315c6c229..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/processors_test.clj+++ /dev/null@@ -1,151 +0,0 @@-(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.processors-test- "Tests for the pure batch processors. Each processor is exercised against an- appdb populated via `mt/with-temp` — no HTTP, no streaming parser. Tests- verify the processor's behavior contract: input shape, return shape, batch- ordering, error attribution, and observable side-effects on the appdb."- (:require- [clojure.test :refer :all]- [metabase-enterprise.serialization.metadata-file-import.processors :as processors]- [metabase.test :as mt]- [toucan2.core :as t2]))--(use-fixtures :once- (fn [thunk]- ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with- ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.- (mt/db)- (mt/with-temporary-setting-values [disable-auto-sync true]- (thunk))))--;;; ============================== with-staging-tables ==============================--(defn- count-staging- "Total row count across both staging tables."- []- (+ (t2/count :metabase_table_import)- (t2/count :metabase_field_import)))--(deftest clear-staging-tables-empties-both-tables-test- (testing "clear-staging-tables! deletes every row from both staging tables"- (t2/insert! :metabase_table_import {:source_id 1 :source_db_id 1 :db_name "x" :name "t"})- (t2/insert! :metabase_field_import {:source_id 1 :source_table_id 1- :name "f" :base_type "type/Integer" :database_type "int"})- (processors/clear-staging-tables!)- (is (zero? (count-staging))- "both staging tables empty after clear-staging-tables!")))--(deftest with-staging-tables-returns-body-value-test- (testing "with-staging-tables yields the value of the body's last form"- (is (= ::sentinel- (processors/with-staging-tables ::sentinel)))))--(deftest with-staging-tables-clears-on-entry-test- (testing "rows present before the macro are gone by the time the body runs"- (t2/insert! :metabase_table_import {:source_id 1 :source_db_id 1 :db_name "leftover" :name "t"})- (t2/insert! :metabase_field_import {:source_id 1 :source_table_id 1- :name "f" :base_type "type/Integer" :database_type "int"})- (let [observed (processors/with-staging-tables (count-staging))]- (is (zero? observed)- "macro entry wipes pre-existing rows so the body sees an empty staging area"))))--(deftest with-staging-tables-clears-on-successful-exit-test- (testing "rows inserted inside the body are gone after the macro returns normally"- (processors/with-staging-tables- (t2/insert! :metabase_table_import {:source_id 1 :source_db_id 1 :db_name "x" :name "t"})- (t2/insert! :metabase_field_import {:source_id 1 :source_table_id 1- :name "f" :base_type "type/Integer" :database_type "int"}))- (is (zero? (count-staging))- "macro exit wipes rows the body added")))--(deftest with-staging-tables-clears-on-thrown-exit-and-propagates-test- (testing "if the body throws, the exception bubbles AND staging tables are still wiped (try/finally)"- (let [thrown (atom nil)]- (try- (processors/with-staging-tables- (t2/insert! :metabase_table_import {:source_id 1 :source_db_id 1 :db_name "x" :name "t"})- (t2/insert! :metabase_field_import {:source_id 1 :source_table_id 1- :name "f" :base_type "type/Integer" :database_type "int"})- (throw (ex-info "boom" {:kind ::test-error})))- (catch clojure.lang.ExceptionInfo e- (reset! thrown e)))- (is (= ::test-error (:kind (ex-data @thrown)))- "exception propagates with its ex-data intact")- (is (zero? (count-staging))- "staging tables wiped even though body threw"))))--;;; ============================== process-databases ==============================--(deftest process-databases-matches-by-name-and-engine-test- (testing "a source row whose (name, engine) pair matches an existing target Database- produces a :matched result whose :target-id is the existing row's id.- :source-id carries the wire :id (source appdb's integer database id);- :name is also surfaced for the orchestrator's db_name denormalization map."- (mt/with-temp [:model/Database {target-id :id} {:name "imported-db" :engine :postgres}]- (is (= [{:source-id 100 :name "imported-db" :target-id target-id :status :matched}]- (into [] (processors/process-databases!- [[1 {:id 100 :name "imported-db" :engine "postgres"}]])))))))--(deftest process-databases-emits-no-match-when-name-or-engine-differs-test- (testing "an unmatched row emits a :no-match result with line attribution and a- human-readable detail string. Mismatches are non-fatal — the loader logs and- skips dependents rather than aborting boot."- (let [[result] (into [] (processors/process-databases!- [[7 {:id 200 :name "no-such-db-zzz" :engine "h2"}]]))]- (is (= 200 (:source-id result)))- (is (= "no-such-db-zzz" (:name result)))- (is (= :no-match (:status result)))- (is (= 7 (:line result)))- (is (string? (:detail result)))- (is (not (contains? result :target-id))- "no :target-id key on no-match (so callers can use (:target-id r) as a presence check)"))))--(deftest process-databases-disambiguates-by-engine-pair-test- (testing "two databases share a name but differ by engine; matching uses the- (name, engine) pair, not just name"- (mt/with-temp [:model/Database {pg-id :id} {:name "shared-name-test" :engine :postgres}- :model/Database {h2-id :id} {:name "shared-name-test" :engine :h2}]- (let [[r] (into [] (processors/process-databases!- [[1 {:id 300 :name "shared-name-test" :engine "h2"}]]))]- (is (= h2-id (:target-id r)))- (is (not= pg-id (:target-id r)))))))--(deftest process-databases-validation-failure-throws-with-attribution-test- (testing "a malformed row (missing required key) throws ex-info carrying- :kind :invalid-input, the file line number, and the row's wire :id- as the :source-id (for attribution in the boot-time error message)."- (let [e (is (thrown? clojure.lang.ExceptionInfo- (into [] (processors/process-databases!- [[42 {:id 400 :name "missing-engine"}]])))) ;; no :engine- data (ex-data e)]- (is (= :invalid-input (:kind data)))- (is (= 42 (:line data)))- (is (= 400 (:source-id data))))))--(deftest process-databases-preserves-input-order-test- (testing "results are in input order regardless of internal SELECT ordering or- which entries match"- (mt/with-temp [:model/Database {a-id :id} {:name "order-a" :engine :postgres}- :model/Database {b-id :id} {:name "order-b" :engine :postgres}]- (let [results (into [] (processors/process-databases!- [[1 {:id 501 :name "order-b" :engine "postgres"}]- [2 {:id 502 :name "no-such-name-q" :engine "h2"}]- [3 {:id 503 :name "order-a" :engine "postgres"}]]))]- (is (= [501 502 503] (mapv :source-id results)))- (is (= ["order-b" "no-such-name-q" "order-a"] (mapv :name results)))- (is (= [b-id nil a-id] (mapv :target-id results)))- (is (= [:matched :no-match :matched] (mapv :status results)))))))--(deftest process-databases-empty-batch-test- (testing "empty input → empty output, no SQL, no exception"- (is (= [] (into [] (processors/process-databases! []))))))--(deftest process-databases-result-is-streamable-test- (testing "the return value supports both `reduce` (the loader's fold path) and- seq/iteration (the test path) without re-running the eager batch work"- (mt/with-temp [:model/Database {target-id :id} {:name "stream-probe" :engine :postgres}]- (let [result (processors/process-databases!- [[1 {:id 700 :name "stream-probe" :engine "postgres"}]])- via-reduce (reduce (fn [acc r] (assoc acc (:source-id r) (:target-id r))) {} result)- via-seq (vec (seq result))]- (is (= {700 target-id} via-reduce))- (is (= 1 (count via-seq)))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/schemas_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/schemas_test.cljdeleted file mode 100644index bbd98ab9bfd..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/schemas_test.clj+++ /dev/null@@ -1,145 +0,0 @@-(ns ^:parallel metabase-enterprise.serialization.metadata-file-import.schemas-test- "Tests for the Malli schemas validating per-line shapes streamed by the- metadata file importer. The schemas describe the wire format- (source-side integer IDs); each row's `:id` is an integer, and cross-row- references (`:db_id`, `:table_id`, `:parent_id`, `:fk_target_field_id`)- are integers pointing at another row's `:id` in the same file.-- The wire-format contract against actual export output is anchored by- `wire_format_test.clj`; these tests focus on schema-level- acceptance/rejection semantics in isolation."- (:require- [clojure.test :refer :all]- [metabase-enterprise.serialization.metadata-file-import.schemas :as schemas]- [metabase.util.malli.registry :as mr]))--(defn- al- "Build a `java.util.ArrayList` from `xs` — mimics what Jackson hands the- importer for JSON arrays."- [xs]- (java.util.ArrayList. ^java.util.List xs))--;;; ============================== ::database-info =====================================--(deftest database-info-accepts-valid-shape-test- (is (mr/validate ::schemas/database-info {:id 7 :name "warehouse" :engine "postgres"})))--(deftest database-info-rejects-missing-required-keys-test- (is (not (mr/validate ::schemas/database-info {:name "warehouse" :engine "postgres"}))- "missing :id")- (is (not (mr/validate ::schemas/database-info {:id 7 :name "warehouse"}))- "missing :engine")- (is (not (mr/validate ::schemas/database-info {:id 7 :engine "postgres"}))- "missing :name")- (is (not (mr/validate ::schemas/database-info {}))))--(deftest database-info-rejects-wrong-types-test- (is (not (mr/validate ::schemas/database-info {:id "not-an-int" :name "warehouse" :engine "postgres"}))- ":id must be an integer")- (is (not (mr/validate ::schemas/database-info {:id 7 :name 5 :engine "postgres"}))- ":name must be a string")- (is (not (mr/validate ::schemas/database-info {:id 7 :name "warehouse" :engine 42}))- ":engine must be a string"))--;;; ============================== ::table-info ========================================--(deftest table-info-accepts-valid-shape-test- (testing "required keys only"- (is (mr/validate ::schemas/table-info- {:id 100 :db_id 7 :name "orders"})))- (testing "with optional :schema and :description"- (is (mr/validate ::schemas/table-info- {:id 100 :db_id 7 :name "orders"- :schema "public" :description "customer orders"})))- (testing ":schema present-and-null is accepted"- (is (mr/validate ::schemas/table-info- {:id 100 :db_id 7 :name "orders" :schema nil}))))--(deftest table-info-rejects-missing-required-keys-test- (is (not (mr/validate ::schemas/table-info {:db_id 7 :name "orders"}))- "missing :id")- (is (not (mr/validate ::schemas/table-info {:id 100 :name "orders"}))- "missing :db_id")- (is (not (mr/validate ::schemas/table-info {:id 100 :db_id 7}))- "missing :name"))--(deftest table-info-rejects-wrong-types-test- (is (not (mr/validate ::schemas/table-info {:id "wh" :db_id 7 :name "orders"}))- ":id must be an integer")- (is (not (mr/validate ::schemas/table-info {:id 100 :db_id "wh" :name "orders"}))- ":db_id must be an integer"))--;;; ============================== ::field-info ========================================--(deftest field-info-accepts-flat-root-field-test- (testing "flat root field — required keys only, no parent or nfc_path"- (is (mr/validate ::schemas/field-info- {:id 1000 :table_id 100 :name "id"- :base_type "type/Integer" :database_type "integer"}))))--(deftest field-info-accepts-row-with-parent-id-and-nfc-path-test- (testing "nested leaf: both :parent_id and :nfc_path set"- (is (mr/validate ::schemas/field-info- {:id 1002 :table_id 100 :name "zip"- :base_type "type/Text" :database_type "text"- :parent_id 1001- :nfc_path ["address"]}))))--(deftest field-info-accepts-fk-target-test- (is (mr/validate ::schemas/field-info- {:id 1003 :table_id 100 :name "user_id"- :base_type "type/Integer" :database_type "integer"- :fk_target_field_id 2000})))--(deftest field-info-accepts-row-with-nfc-path-and-no-parent-id-test- (testing "unfolded leaf: :nfc_path set, no :parent_id (no parent storage row exists)"- (is (mr/validate ::schemas/field-info- {:id 1004 :table_id 100 :name "payload → address → zip"- :base_type "type/Text" :database_type "text"- :nfc_path ["payload" "address" "zip"]})))- (testing "Jackson ArrayList for :nfc_path also validates"- (is (mr/validate ::schemas/field-info- {:id 1004 :table_id 100 :name "payload → address → zip"- :base_type "type/Text" :database_type "text"- :nfc_path (al ["payload" "address" "zip"])}))))--(deftest field-info-accepts-all-optional-keys-test- (is (mr/validate ::schemas/field-info- {:id 1005 :table_id 100 :name "amount"- :base_type "type/Float" :database_type "DECIMAL"- :description "order total"- :effective_type "type/Float"- :semantic_type "type/Currency"- :coercion_strategy "Coercion/UNIXSeconds->DateTime"- :parent_id 1001- :fk_target_field_id 2000- :nfc_path ["amount"]})))--(deftest field-info-rejects-missing-required-keys-test- (let [base {:id 1000 :table_id 100 :name "x"- :base_type "type/Integer" :database_type "integer"}]- (is (not (mr/validate ::schemas/field-info (dissoc base :id))) "missing :id")- (is (not (mr/validate ::schemas/field-info (dissoc base :table_id))) "missing :table_id")- (is (not (mr/validate ::schemas/field-info (dissoc base :name))) "missing :name")- (is (not (mr/validate ::schemas/field-info (dissoc base :base_type))) "missing :base_type")- (is (not (mr/validate ::schemas/field-info (dissoc base :database_type))) "missing :database_type")))--(deftest field-info-rejects-wrong-id-types-test- (let [base {:id 1000 :table_id 100 :name "x"- :base_type "type/Integer" :database_type "integer"}]- (is (not (mr/validate ::schemas/field-info (assoc base :id "not-int")))- ":id must be an integer")- (is (not (mr/validate ::schemas/field-info (assoc base :table_id "not-int")))- ":table_id must be an integer")- (is (not (mr/validate ::schemas/field-info (assoc base :parent_id "not-int")))- ":parent_id (when present) must be an integer")- (is (not (mr/validate ::schemas/field-info (assoc base :fk_target_field_id "not-int")))- ":fk_target_field_id (when present) must be an integer")))--(deftest field-info-rejects-malformed-nfc-path-test- (let [base {:id 1000 :table_id 100 :name "x"- :base_type "type/Integer" :database_type "integer"}]- (is (not (mr/validate ::schemas/field-info (assoc base :nfc_path "not-a-list")))- ":nfc_path must be a list/vector")- (is (not (mr/validate ::schemas/field-info (assoc base :nfc_path [1 2 3])))- ":nfc_path elements must be strings")))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/tables_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/tables_test.cljdeleted file mode 100644index 8b44a1416fc..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/tables_test.clj+++ /dev/null@@ -1,259 +0,0 @@-(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.tables-test- "Tests for `resolve-target-table-ids-in-staging!` and `merge-tables!` —- the table-merge half of the pipeline. These functions key on the- natural-key tuple (db_name, schema, name) against `metabase_table`,- filtered to active non-defective rows."- (:require- [clojure.string :as str]- [clojure.test :refer :all]- [metabase-enterprise.serialization.metadata-file-import.processors :as p]- [metabase.test :as mt]- [toucan2.core :as t2]))--(use-fixtures :once- (fn [thunk]- ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with- ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.- (mt/db)- (mt/with-temporary-setting-values [disable-auto-sync true]- (thunk))))--(defn- staging-row [source-id db-name name & {:as overrides}]- (merge {:source_id source-id- :source_db_id 1- :db_name db-name- :schema "PUBLIC"- :name name- :display_name (str/capitalize name)}- overrides))--(defn- staging-rows-by-source-id []- ;; HoneySQL (not raw SQL) so `schema` is dialect-quoted correctly on MySQL.- (into {} (map (juxt :source_id identity))- (t2/query {:select [:source_id :db_name :schema :name :description :target_id]- :from [:metabase_table_import]})))--;;; ============================== resolve-target-table-ids ==============================--(deftest resolve-populates-target-id-when-match-exists-test- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"}]- (try- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])- (p/resolve-target-table-ids-in-staging!)- (let [row (get (staging-rows-by-source-id) 100)]- (is (= target-id (:target_id row))))- (finally (p/clear-staging-tables!)))))--(deftest resolve-leaves-target-id-null-when-no-match-test- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {} {:db_id db-id :schema "PUBLIC" :name "users"}]- (try- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 db-name "no_such_table")])- (p/resolve-target-table-ids-in-staging!)- (let [row (get (staging-rows-by-source-id) 100)]- (is (nil? (:target_id row))))- (finally (p/clear-staging-tables!)))))--(deftest resolve-handles-nil-schema-test- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {target-id :id} {:db_id db-id :schema nil :name "raw_table"}]- (try- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 db-name "raw_table" :schema nil)])- (p/resolve-target-table-ids-in-staging!)- (let [row (get (staging-rows-by-source-id) 100)]- (is (= target-id (:target_id row))- "NULL-safe match: staging schema=NULL ⇔ target schema=NULL"))- (finally (p/clear-staging-tables!)))))--(deftest resolve-matches-inactive-target-test- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {tid :id} {:db_id db-id :schema "PUBLIC" :name "users" :active false}]- (try- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])- (p/resolve-target-table-ids-in-staging!)- (let [row (get (staging-rows-by-source-id) 100)]- (is (= tid (:target_id row))- "inactive (non-defective) target rows resolve to the existing row — the merge reactivates- it in place rather than inserting a duplicate that collides on the unique index"))- (finally (p/clear-staging-tables!)))))--(deftest resolve-skips-defective-duplicate-target-test- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {} {:db_id db-id :schema "PUBLIC" :name "users" :is_defective_duplicate true}]- (try- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])- (p/resolve-target-table-ids-in-staging!)- (let [row (get (staging-rows-by-source-id) 100)]- (is (nil? (:target_id row))))- (finally (p/clear-staging-tables!)))))--(deftest resolve-is-idempotent-test- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"}]- (try- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])- (p/resolve-target-table-ids-in-staging!)- (p/resolve-target-table-ids-in-staging!)- (let [row (get (staging-rows-by-source-id) 100)]- (is (= target-id (:target_id row))))- (finally (p/clear-staging-tables!)))))--(deftest resolve-tolerates-duplicate-natural-key-target-test- (testing "If two live (db_name, schema, name) candidates exist — possible- because `metabase_database`'s unique constraint is- `(router_database_id, name)` and NULL routers don't collide —- the resolve subquery picks a stable tiebreaker (MIN(id)) rather- than throwing `subquery returned more than one row`. Regression- for GHY-3549."- (mt/with-temp [:model/Database {db-id-1 :id db-name :name} {:engine :h2}- :model/Database {db-id-2 :id} {:engine :h2}]- ;; Make the second Database share the first's name; mt/with-temp gives- ;; each a unique name but PG's unique constraint is `(router_database_id,- ;; name)`, so an UPDATE that sets a duplicate (NULL, name) is permitted.- (t2/query {:update :metabase_database :set {:name db-name} :where [:= :id db-id-2]})- (let [t1-id (:id (t2/insert-returning-instance!- :model/Table {:db_id db-id-1 :schema "PUBLIC" :name "users"- :display_name "Users" :active true}))- t2-id (:id (t2/insert-returning-instance!- :model/Table {:db_id db-id-2 :schema "PUBLIC" :name "users"- :display_name "Users" :active true}))]- (try- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 db-name "users")])- ;; The bug pre-fix: resolve throws "subquery returned more than one row".- ;; With the fix: resolve completes and target_id is set to MIN(id).- (p/resolve-target-table-ids-in-staging!)- (let [row (get (staging-rows-by-source-id) 100)]- (is (= (min t1-id t2-id) (:target_id row))- "MIN(id) tiebreaker: target_id is the smaller of the two candidates"))- (finally- (p/clear-staging-tables!)))))))--;;; ============================== merge-tables ==============================--(deftest merge-empty-staging-is-noop-test- (try- (p/clear-staging-tables!)- (let [before (t2/count :metabase_table)]- (p/merge-tables!)- (is (= before (t2/count :metabase_table))))- (finally (p/clear-staging-tables!))))--(deftest merge-inserts-unmatched-table-test- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}]- (try- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 db-name "new_table"- :description "first import")])- (p/resolve-target-table-ids-in-staging!)- (p/merge-tables!)- (let [inserted (t2/select-one :model/Table :db_id db-id :name "new_table")]- (is (some? inserted))- (is (= "PUBLIC" (:schema inserted)))- (is (= "first import" (:description inserted)))- (is (true? (:active inserted)))- (is (= "internal" (name (:data_layer inserted)))))- (finally- (t2/delete! :model/Table :db_id db-id)- (p/clear-staging-tables!)))))--(deftest merge-updates-description-on-match-test- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"- :description "old description"}]- (try- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 db-name "users"- :description "new description")])- (p/resolve-target-table-ids-in-staging!)- (p/merge-tables!)- (is (= "new description" (:description (t2/select-one :model/Table :id target-id))))- (finally (p/clear-staging-tables!)))))--(deftest merge-skips-row-whose-db-has-no-target-test- (try- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 "nonexistent_db" "ghost_table")])- (p/resolve-target-table-ids-in-staging!)- (let [before (t2/count :model/Table :name "ghost_table")]- (p/merge-tables!)- (is (= before (t2/count :model/Table :name "ghost_table"))- "INSERT JOINs metabase_database on db_name; no match → row silently skipped"))- (finally (p/clear-staging-tables!))))--(deftest merge-handles-nil-schema-test- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}]- (try- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 db-name "raw_table" :schema nil)])- (p/resolve-target-table-ids-in-staging!)- (p/merge-tables!)- (let [inserted (t2/select-one :model/Table :db_id db-id :name "raw_table")]- (is (some? inserted))- (is (nil? (:schema inserted))))- (finally- (t2/delete! :model/Table :db_id db-id)- (p/clear-staging-tables!)))))--(deftest merge-is-idempotent-test- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}]- (try- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 db-name "users" :description "v1")])- (p/resolve-target-table-ids-in-staging!)- (p/merge-tables!)- (let [first-id (:id (t2/select-one :model/Table :db_id db-id :name "users"))]- ;; second pass — re-resolve and re-merge against the same staging contents- (p/resolve-target-table-ids-in-staging!)- (p/merge-tables!)- (let [second-id (:id (t2/select-one :model/Table :db_id db-id :name "users"))]- (is (= first-id second-id) "re-import doesn't insert a duplicate")- (is (= 1 (t2/count :model/Table :db_id db-id :name "users")))))- (finally- (t2/delete! :model/Table :db_id db-id)- (p/clear-staging-tables!)))))--(deftest merge-skip-if-unchanged-test- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"- :description "same"}]- (try- ;; Stamp updated_at to a known-old value so we can detect if the UPDATE fires.- (t2/query {:update :metabase_table- :set {:updated_at #t "2020-01-01T00:00:00Z"}- :where [:= :id target-id]})- (let [original-updated-at (:updated_at (t2/select-one :model/Table :id target-id))]- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 db-name "users" :description "same")])- (p/resolve-target-table-ids-in-staging!)- (p/merge-tables!)- (let [after-updated-at (:updated_at (t2/select-one :model/Table :id target-id))]- (is (= original-updated-at after-updated-at)- "skip-if-unchanged: identical description means no UPDATE fires, updated_at unchanged")))- (finally (p/clear-staging-tables!)))))--(deftest merge-fires-when-description-differs-test- (mt/with-temp [:model/Database {db-id :id db-name :name} {:engine :h2}- :model/Table {target-id :id} {:db_id db-id :schema "PUBLIC" :name "users"- :description "v1"}]- (try- (t2/query {:update :metabase_table- :set {:updated_at #t "2020-01-01T00:00:00Z"}- :where [:= :id target-id]})- (let [original-updated-at (:updated_at (t2/select-one :model/Table :id target-id))]- (p/clear-staging-tables!)- (t2/insert! :metabase_table_import [(staging-row 100 db-name "users" :description "v2")])- (p/resolve-target-table-ids-in-staging!)- (p/merge-tables!)- (let [after (t2/select-one :model/Table :id target-id)]- (is (= "v2" (:description after)))- (is (not= original-updated-at (:updated_at after))- "description differs → UPDATE fires → updated_at bumped")))- (finally (p/clear-staging-tables!)))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/wire_format_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/wire_format_test.cljdeleted file mode 100644index 661b97ef9b5..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import/wire_format_test.clj+++ /dev/null@@ -1,132 +0,0 @@-(ns ^:synchronous metabase-enterprise.serialization.metadata-file-import.wire-format-test- "Contract test between the metadata export and import sides. The importer's- Malli schemas describe what the importer expects on the wire; this test- exercises the export pipeline against a small hand-built fixture and- validates every emitted row against the corresponding import schema. If the- export starts emitting a column the import schema doesn't know about — or- stops emitting a required one — the test fails.-- NOT `^:parallel`: each test does an `mt/with-temp [:model/Database :model/Table- :model/Field …]` whose `:after-insert` chain hits `metabase_data_permissions`- + the cluster lock. Concurrent tests in this ns reliably deadlock on MariaDB."- (:require- [clojure.test :refer :all]- [metabase-enterprise.serialization.export :as export]- [metabase-enterprise.serialization.metadata-file-import.schemas :as schemas]- [metabase.test :as mt]- [metabase.util.json :as json]- [metabase.util.malli.registry :as mr])- (:import- (java.io ByteArrayOutputStream)))--(set! *warn-on-reflection* true)--(use-fixtures :once- (fn [thunk]- ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with- ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.- (mt/db)- (mt/with-temporary-setting-values [disable-auto-sync true]- (thunk))))--(defn- export-as-json!- "Run the export against `opts`, decode JSON to keyword-keyed maps."- [opts]- (let [bos (ByteArrayOutputStream.)]- (export/export-metadata! bos opts)- (json/decode+kw (.toString bos "UTF-8"))))--(defmacro ^:private with-tiny-fixture!- "Bind a hand-built fixture covering each wire-format field shape:- - flat root field (no parent, no nfc_path, no FK)- - Dictionary parent (has children pointing at it)- - nested leaf (parent_id + nfc_path)- - unfolded leaf (nfc_path only, no parent_id)- - FK source (fk_target_field_id pointing at another table's field)- - FK target (referenced from another field via fk_target_field_id)-- Yields `[db-id table-1-id table-2-id]` in scope of `body`."- [bindings & body]- (let [[db table1 table2] bindings]- `(mt/with-temp- [:model/Database {~db :id} {:engine :h2}- :model/Table {~table1 :id} {:db_id ~db :schema "PUBLIC" :name "users"}- :model/Table {~table2 :id} {:db_id ~db :schema "PUBLIC" :name "accounts"}- :model/Field _flat# {:table_id ~table1 :name "user_id" :base_type :type/Integer}- :model/Field {parent-id# :id} {:table_id ~table1 :name "data" :base_type :type/Dictionary}- :model/Field _nested-leaf# {:table_id ~table1 :name "value"- :base_type :type/Text- :parent_id parent-id#- :nfc_path ["data" "value"]}- :model/Field _unfolded-leaf# {:table_id ~table1 :name "embedded → leaf"- :base_type :type/Text- :nfc_path ["embedded" "leaf"]}- :model/Field {target-id# :id} {:table_id ~table2 :name "id" :base_type :type/Integer}- :model/Field _fk-source# {:table_id ~table1- :name "account_fk"- :base_type :type/Integer- :fk_target_field_id target-id#}]- ~@body)))--(defn- export-fixture!- "Run the export scoped to the fixture's database id."- [db-id]- (export-as-json! {:user-info {:user-id (mt/user->id :crowberto) :is-superuser? true}- :with-databases true- :with-tables true- :with-fields true- :database-ids [db-id]}))--;;; ============================== Per-section tests ==============================--(deftest database-rows-validate-against-import-schema-test- (mt/with-premium-features #{:serialization}- (with-tiny-fixture! [db-id _t1 _t2]- (let [{:keys [databases]} (export-fixture! db-id)]- (testing "fixture surfaces at least one database row"- (is (= 1 (count databases))))- (doseq [row databases]- (testing (str "row validates against ::schemas/database-info — " (pr-str row))- (is (mr/validate ::schemas/database-info row))))))))--(deftest table-rows-validate-against-import-schema-test- (mt/with-premium-features #{:serialization}- (with-tiny-fixture! [db-id _t1 _t2]- (let [{:keys [tables]} (export-fixture! db-id)]- (testing "fixture surfaces both tables"- (is (= 2 (count tables))))- (doseq [row tables]- (testing (str "row validates against ::schemas/table-info — " (pr-str row))- (is (mr/validate ::schemas/table-info row))))))))--(deftest field-rows-validate-against-import-schema-test- (mt/with-premium-features #{:serialization}- (with-tiny-fixture! [db-id _t1 _t2]- (let [{:keys [fields]} (export-fixture! db-id)]- (testing "fixture surfaces all six field shapes"- (is (= 6 (count fields))))- (doseq [row fields]- (testing (str "row validates against ::schemas/field-info — " (pr-str row))- (is (mr/validate ::schemas/field-info row))))))))--;;; ============================== Per-shape coverage assertions ==============================-;;; Belt-and-suspenders: assert the fixture actually exercised each variant.-;;; If the fixture stops producing a particular shape (e.g., a refactor of-;;; mt/with-temp's defaults), the per-section tests still pass, but these-;;; coverage tests catch the gap.--(deftest field-shape-coverage-test- (mt/with-premium-features #{:serialization}- (with-tiny-fixture! [db-id _t1 _t2]- (let [{:keys [fields]} (export-fixture! db-id)]- (testing "exactly one nested leaf (parent_id + nfc_path)"- (is (= 1 (count (filter #(and (:parent_id %) (:nfc_path %)) fields)))))- (testing "exactly one unfolded leaf (nfc_path without parent_id)"- (is (= 1 (count (filter #(and (:nfc_path %) (not (:parent_id %))) fields)))))- (testing "exactly one FK source (fk_target_field_id present)"- (is (= 1 (count (filter :fk_target_field_id fields)))))- (testing "at least one flat field (no parent_id, no nfc_path, no FK)"- (is (pos? (count (filter #(and (not (:parent_id %))- (not (:nfc_path %))- (not (:fk_target_field_id %)))- fields)))))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import_test.cljdeleted file mode 100644index 59c59e4bc93..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/metadata_file_import_test.clj+++ /dev/null@@ -1,279 +0,0 @@-(ns metabase-enterprise.serialization.metadata-file-import-test- "End-to-end orchestrator tests for [[metabase-enterprise.serialization.metadata-file-import/import-metadata-file!]].- Each test writes a temp JSON file in the wire format, sets up matching- `mt/with-temp` Database/Table/Field rows where needed, runs the loader, and- asserts on the appdb state. See- [[metabase-enterprise.serialization.metadata-file-import.schemas]] for the- wire-format primer."- (:require- [clojure.test :refer :all]- [metabase-enterprise.serialization.metadata-file-import :as loader]- [metabase-enterprise.serialization.metadata-file-import.processors :as processors]- [metabase.test :as mt]- [metabase.util.json :as json]- [metabase.warehouse-schema.models.table :as schema.table]- [toucan2.core :as t2])- (:import- (java.io File)))--(set! *warn-on-reflection* true)--;; NOT ^:parallel — test-helpers-set-global-values! asserts non-parallel.-;; See note in concurrency_test.clj fixture — intentional opt-out from the-;; parallel-safe contract that kondo's `validate-deftest` enforces.-#_{:clj-kondo/ignore [:metabase/validate-deftest]}-(use-fixtures :once- (fn [thunk]- ;; Warm test-data (load + sync) before disabling auto-sync: the first load must not happen with- ;; sync off, or with-temp defaults that resolve test-data tables (e.g. `(data/id :checkins)`) fail.- (mt/db)- (mt/with-temporary-setting-values [disable-auto-sync true]- (mt/test-helpers-set-global-values!- (thunk)))))--;;; ============================== Test helpers ==============================--(defn- temp-file ^File [suffix content]- (let [f (File/createTempFile "loader-test-" suffix)]- (.deleteOnExit f)- (spit f content)- f))--(defn- json-file ^File [data]- (temp-file ".json" (json/encode data)))--;;; ============================== Happy paths ==============================--(deftest end-to-end-happy-path-with-json-file-test- (testing "given a target Database that matches by (name, engine) and a metadata file- describing one table with one root field and one nested field, the loader- inserts the missing rows and updates initial_sync_status on the matched DB"- (mt/with-temp [:model/Database {tgt-db :id} {:name "happy-path-db" :engine :postgres- :initial_sync_status "incomplete"}]- (let [meta-file (json-file- {:databases [{:id 7 :name "happy-path-db" :engine "postgres"}]- :tables [{:id 100 :db_id 7 :schema "public" :name "orders"}]- :fields [{:id 1000 :table_id 100 :name "id"- :base_type "type/Integer" :database_type "integer"}- {:id 1001 :table_id 100 :name "address"- :base_type "type/Structured" :database_type "json"}- {:id 1002 :table_id 100 :name "zip"- :parent_id 1001 :nfc_path ["address"]- :base_type "type/Text" :database_type "text"}]})]- (loader/import-metadata-file! meta-file)- (testing "the matched Database's initial_sync_status was flipped to complete"- (is (= "complete" (:initial_sync_status (t2/select-one :model/Database :id tgt-db)))))- (testing "the table was inserted, attached to the target DB"- (let [tbl (t2/select-one :model/Table :db_id tgt-db :name "orders")]- (is (some? tbl))- (is (= "public" (:schema tbl)))))- (testing "the root and nested fields were inserted with correct parent linkage"- (let [tbl-id (:id (t2/select-one :model/Table :db_id tgt-db :name "orders"))- root (t2/select-one :model/Field :table_id tbl-id :name "id")- addr (t2/select-one :model/Field :table_id tbl-id :name "address")- zip (t2/select-one :model/Field :table_id tbl-id :name "zip")]- (is (some? root))- (is (some? addr))- (is (some? zip))- (is (nil? (:parent_id root)))- (is (nil? (:parent_id addr)))- (is (= (:id addr) (:parent_id zip))- "the nested field's parent_id is the address field's target id")- (is (= ["address"] (:nfc_path zip)))))))))--;;; ============================== No-match skipping ==============================--(deftest unmatched-source-database-skips-its-tables-and-fields-test- (testing "a source database with no (name, engine) match in target: WARN logged,- its tables/fields are silently dropped during merge"- (let [meta-file (json-file- {:databases [{:id 7 :name "no-such-db" :engine "postgres"}]- :tables [{:id 100 :db_id 7 :schema "public" :name "ghost"}]- :fields [{:id 1000 :table_id 100 :name "x"- :base_type "type/Integer" :database_type "int"}]})]- (loader/import-metadata-file! meta-file)- (is (zero? (t2/count :model/Table :name "ghost"))- "no metabase_table row was created for the unmatched-DB's table")- (is (zero? (t2/count :model/Field :name "x"))- "no metabase_field row either"))))--;;; ============================== FK resolution ==============================--(deftest fk-target-field-id-resolved-after-import-test- (testing "two tables, one with a field that has fk_target_field_id pointing at the- other table's id field — after import, target appdb's fk_target_field_id- points at the correctly-resolved target field id"- (mt/with-temp [:model/Database {tgt-db :id} {:name "fk-test-db" :engine :postgres}]- (let [meta-file (json-file- {:databases [{:id 7 :name "fk-test-db" :engine "postgres"}]- :tables [{:id 100 :db_id 7 :schema "public" :name "users"}- {:id 101 :db_id 7 :schema "public" :name "orders"}]- :fields [{:id 1000 :table_id 100 :name "id"- :base_type "type/Integer" :database_type "integer"}- {:id 1001 :table_id 101 :name "user_id"- :base_type "type/Integer" :database_type "integer"- :fk_target_field_id 1000}]})]- (loader/import-metadata-file! meta-file)- (let [users-tbl (:id (t2/select-one :model/Table :db_id tgt-db :name "users"))- orders-tbl (:id (t2/select-one :model/Table :db_id tgt-db :name "orders"))- users-id (:id (t2/select-one :model/Field :table_id users-tbl :name "id"))- orders-fk (t2/select-one :model/Field :table_id orders-tbl :name "user_id")]- (is (= users-id (:fk_target_field_id orders-fk))- "orders.user_id's fk_target_field_id is users.id's resolved target id"))))))--;;; ============================== Per-table default permissions ==============================--(deftest newly-imported-tables-get-default-permissions-test- (testing "Tables inserted by the importer flow through `set-new-tables-permissions!`- in batch, mirroring what the `:after-insert` hook on `:model/Table`- would do for single-row inserts (just in one call instead of N).- Verified via a spy on the bulk grant function — asserting which- rows actually land in `data_permissions` would couple the test to- permission-graph semantics (db-level perms cover new tables ⇒ no- table-level rows), which isn't what we're checking here."- (mt/with-temp [:model/Database {tgt-db :id} {:name "perms-bulk-db" :engine :postgres}]- (let [meta-file (json-file- {:databases [{:id 7 :name "perms-bulk-db" :engine "postgres"}]- :tables [{:id 100 :db_id 7 :schema "public" :name "alpha"}- {:id 101 :db_id 7 :schema "public" :name "beta"}]- :fields [{:id 1000 :table_id 100 :name "id"- :base_type "type/Integer" :database_type "integer"}- {:id 1001 :table_id 101 :name "id"- :base_type "type/Integer" :database_type "integer"}]})- spy (atom [])- real-bulk @#'schema.table/set-new-tables-permissions!]- (with-redefs [schema.table/set-new-tables-permissions!- (fn [db-id rows]- (swap! spy conj {:db-id db-id :rows rows})- (real-bulk db-id rows))]- (loader/import-metadata-file! meta-file))- (let [calls @spy]- (is (= 1 (count calls)) "set-new-tables-permissions! fired exactly once (per db_id)")- (let [{:keys [db-id rows]} (first calls)]- (is (= tgt-db db-id) "called with the target db_id")- (is (= 2 (count rows)) "passed both newly-inserted tables")- (is (= #{"alpha" "beta"}- (set (map :name- (t2/select [:model/Table :name] :id [:in (map :id rows)]))))- "rows correspond to alpha and beta")))))))--;;; ============================== Idempotence ==============================--(deftest re-import-is-idempotent-test- (testing "running the same file twice produces the same final appdb state- — no duplicate rows, no error"- (mt/with-temp [:model/Database {tgt-db :id} {:name "idem-db" :engine :postgres}]- (let [meta-file (json-file- {:databases [{:id 7 :name "idem-db" :engine "postgres"}]- :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]- :fields [{:id 1000 :table_id 100 :name "x"- :base_type "type/Integer" :database_type "int"- :description "v1"}]})]- (loader/import-metadata-file! meta-file)- (loader/import-metadata-file! meta-file)- (let [tbl-id (:id (t2/select-one :model/Table :db_id tgt-db :name "t"))]- (is (= 1 (t2/count :model/Table :db_id tgt-db :name "t")))- (is (= 1 (t2/count :model/Field :table_id tbl-id :name "x")))- (is (= "v1" (:description (t2/select-one :model/Field :table_id tbl-id :name "x")))))))))--(deftest re-import-reactivates-deactivated-table-test- (testing "re-importing a table that was deactivated after the first import reactivates it in place- rather than inserting a duplicate (which violates idx_uniq_table_db_id_schema_name). The- field path already matches + reactivates regardless of `active`; tables must too."- (mt/with-temp [:model/Database {tgt-db :id} {:name "reactivate-db" :engine :postgres}]- (let [meta-file (json-file- {:databases [{:id 7 :name "reactivate-db" :engine "postgres"}]- :tables [{:id 100 :db_id 7 :schema "public" :name "orders"}]- :fields [{:id 1000 :table_id 100 :name "id"- :base_type "type/Integer" :database_type "integer"}]})]- (loader/import-metadata-file! meta-file)- (let [tbl-id (:id (t2/select-one :model/Table :db_id tgt-db :name "orders"))]- ;; simulate sync (or a prior run) deactivating the table after the first import- (t2/update! :model/Table tbl-id {:active false})- ;; re-import the same file: must reactivate in place, not insert a duplicate- (loader/import-metadata-file! meta-file)- (testing "no duplicate row was inserted (would otherwise collide on the unique index)"- (is (= 1 (t2/count :model/Table :db_id tgt-db :schema "public" :name "orders"))))- (testing "the existing row was reactivated in place (same id, active=true)"- (let [tbl (t2/select-one :model/Table :db_id tgt-db :schema "public" :name "orders")]- (is (= tbl-id (:id tbl)))- (is (true? (:active tbl))))))))))--;;; ============================== Pre-flight orphan bail ==============================--(deftest pre-flight-orphan-bail-test- (testing "a file whose field references a parent_id not in the same file is rejected- with :file-incomplete; no live writes happen"- (mt/with-temp [:model/Database {} {:name "orphan-bail-db" :engine :postgres}]- (let [meta-file (json-file- {:databases [{:id 7 :name "orphan-bail-db" :engine "postgres"}]- :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]- :fields [{:id 1000 :table_id 100 :name "x"- :base_type "type/Integer" :database_type "int"- :parent_id 9999}]}) ; bad ref — no such source_id in file- thrown (try (loader/import-metadata-file! meta-file) nil- (catch clojure.lang.ExceptionInfo e e))]- (is (some? thrown))- (is (= :file-incomplete (:kind (ex-data thrown))))- (is (zero? (t2/count :model/Table :name "t"))- "no live writes happened — pre-flight ran before any txn")))))--(deftest cycle-bail-test- (testing "a file with a cycle in source_parent_id is rejected with- :cycle-in-field-graph; no live writes happen"- (mt/with-temp [:model/Database {} {:name "cycle-db" :engine :postgres}]- (let [meta-file (json-file- {:databases [{:id 7 :name "cycle-db" :engine "postgres"}]- :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]- :fields [{:id 1000 :table_id 100 :name "a"- :base_type "type/Integer" :database_type "int"- :parent_id 1001}- {:id 1001 :table_id 100 :name "b"- :base_type "type/Integer" :database_type "int"- :parent_id 1000}]})- thrown (try (loader/import-metadata-file! meta-file) nil- (catch clojure.lang.ExceptionInfo e e))]- (is (some? thrown))- (is (= :cycle-in-field-graph (:kind (ex-data thrown))))- (is (zero? (t2/count :model/Table :name "t")))))))--;;; ============================== Atomic rollback ==============================--(deftest atomic-rollback-on-mid-merge-throw-test- (testing "if anything inside the merge transaction throws, all live writes are rolled- back — neither tables nor fields land in the appdb"- (mt/with-temp [:model/Database {tgt-db :id} {:name "rollback-db" :engine :postgres}]- (let [meta-file (json-file- {:databases [{:id 7 :name "rollback-db" :engine "postgres"}]- :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]- :fields [{:id 1000 :table_id 100 :name "x"- :base_type "type/Integer" :database_type "int"}]})]- ;; Inject a throw into merge-fields-by-depth! — runs after merge-tables! has- ;; written to metabase_table.- (with-redefs [processors/merge-fields-by-depth!- (fn [] (throw (ex-info "boom mid-merge" {:kind ::injected})))]- (let [thrown (try (loader/import-metadata-file! meta-file) nil- (catch clojure.lang.ExceptionInfo e e))]- (is (some? thrown))- (is (= ::injected (:kind (ex-data thrown))))))- (is (zero? (t2/count :model/Table :db_id tgt-db :name "t"))- "the table that merge-tables! 'inserted' was rolled back")))))--;;; ============================== field-shape coverage ==============================--(deftest unfolded-leaf-imports-correctly-test- (testing "an unfolded leaf (nfc_path set, no parent_id) imports as a leaf field- with nfc_path preserved and parent_id NULL"- (mt/with-temp [:model/Database {tgt-db :id} {:name "unfolded-leaf-db" :engine :postgres}]- (let [meta-file (json-file- {:databases [{:id 7 :name "unfolded-leaf-db" :engine "postgres"}]- :tables [{:id 100 :db_id 7 :schema "public" :name "t"}]- :fields [{:id 1000 :table_id 100 :name "data.user.zip"- :base_type "type/Text" :database_type "text"- :nfc_path ["data" "user" "zip"]}]})]- (loader/import-metadata-file! meta-file)- (let [tbl-id (:id (t2/select-one :model/Table :db_id tgt-db :name "t"))- leaf (t2/select-one :model/Field :table_id tbl-id :name "data.user.zip")]- (is (some? leaf))- (is (nil? (:parent_id leaf)))- (is (= ["data" "user" "zip"] (:nfc_path leaf))))))))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api/workspace_instance_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api/workspace_instance_test.cljindex d219bdca5b3..c77466279a7 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/api/workspace_instance_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api/workspace_instance_test.clj@@ -4,7 +4,8 @@request/response shape through the HTTP layer."(:require[clojure.test :refer [deftest is testing use-fixtures]]- [metabase-enterprise.workspaces.instance :as ws.instance]+ [metabase-enterprise.workspaces.core :as ws]+ [metabase-enterprise.workspaces.test-util :as ws.tu][metabase.test :as mt][metabase.test.fixtures :as fixtures][toucan2.core :as t2]))@@ -17,7 +18,7 @@(try(thunk)(finally- (ws.instance/clear-instance-workspace!))))))+ (ws/clear-instance-workspace!))))))(deftest current-superuser-only-test(testing "GET /ee/workspace-instance/current requires data analyst or superuser"@@ -26,18 +27,20 @@(deftest current-empty-test(testing "GET /ee/workspace-instance/current returns {:data nil} when no workspace is loaded"- (with-redefs [ws.instance/instance-workspace (constantly nil)]+ (with-redefs [ws/instance-workspace (constantly nil)](is (= {:data nil}(mt/user-http-request :crowberto :get 200 "ee/workspace-instance/current"))))))(deftest current-with-workspace-test(testing "GET /ee/workspace-instance/current returns the workspace wrapped in {:data ...}"- (with-redefs [ws.instance/instance-workspace (constantly {:name "Current Test"- :databases {(mt/id) {:input_schemas ["PUBLIC"]- :output {:schema "ws_alice"}}}})]+ (with-redefs [ws/instance-workspace (constantly {:name "Current Test"+ :databases {(mt/id) {:input_schemas ["PUBLIC"]+ :output {:schema "ws_alice"}}}})](let [result (mt/user-http-request :crowberto :get 200 "ee/workspace-instance/current")data (:data result)](is (= "Current Test" (:name data)))+ (testing "can_write is true when the workspace is not locked by config"+ (is (true? (:can_write data))))(testing "databases is a map keyed by database id with input_schemas + output"(is (= 1 (count (:databases data))))(let [databases (:databases data)@@ -47,6 +50,111 @@(is (= ["PUBLIC"] (:input_schemas db)))(is (= {:schema "ws_alice"} (:output db)))))))))+(deftest current-with-workspace-locked-returns-can-write-false-test+ (testing "GET /ee/workspace-instance/current returns can_write false when locked by config"+ (ws.tu/with-workspace-locked-by-config+ (fn []+ (with-redefs [ws/instance-workspace (constantly {:name "Locked Test" :databases {}})]+ (let [data (:data (mt/user-http-request :crowberto :get 200 "ee/workspace-instance/current"))]+ (is (= "Locked Test" (:name data)))+ (is (false? (:can_write data)))))))))++(deftest current-with-workspace-locked-by-env-var-returns-can-write-false-test+ (testing "GET /ee/workspace-instance/current returns can_write false when MB_INSTANCE_WORKSPACE is set"+ (mt/with-temp-env-var-value! [:mb-instance-workspace "{\"name\":\"x\",\"databases\":{}}"]+ (with-redefs [ws/instance-workspace (constantly {:name "Env Locked Test" :databases {}})]+ (let [data (:data (mt/user-http-request :crowberto :get 200 "ee/workspace-instance/current"))]+ (is (= "Env Locked Test" (:name data)))+ (is (false? (:can_write data))))))))++(deftest post-current-superuser-only-test+ (testing "POST /ee/workspace-instance/current requires data analyst or superuser"+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :post 403 "ee/workspace-instance/current"+ {:name "x" :databases {}})))))++(deftest post-current-round-trip-test+ (testing "POST /ee/workspace-instance/current installs the workspace and is readable via GET"+ (let [payload {:name "Installed via API"+ :databases {(mt/id) {:input_schemas ["PUBLIC"]+ :output {:schema "ws_api_install"}}}}+ posted (mt/user-http-request :crowberto :post 200 "ee/workspace-instance/current" payload)]+ (is (= "Installed via API" (:name posted)))+ (testing "the POST response reports can_write true while unlocked"+ (is (true? (:can_write posted))))+ (testing "GET reflects the workspace just installed"+ (let [fetched (:data (mt/user-http-request :crowberto :get 200 "ee/workspace-instance/current"))]+ (is (= "Installed via API" (:name fetched)))+ (is (= 1 (count (:databases fetched)))))))))++(deftest post-current-rejects-when-locked-test+ (testing "POST /ee/workspace-instance/current returns 400 when locked by config"+ (ws.tu/with-workspace-locked-by-config+ (fn []+ (let [resp (mt/user-http-request :crowberto :post 400 "ee/workspace-instance/current"+ {:name "x" :databases {}})]+ (is (re-find #"config\.yml" (:message resp))))))))++(deftest post-current-rejects-when-env-var-set-test+ (testing "POST /ee/workspace-instance/current returns 400 when MB_INSTANCE_WORKSPACE is set"+ (mt/with-temp-env-var-value! [:mb-instance-workspace "{\"name\":\"x\",\"databases\":{}}"]+ (let [resp (mt/user-http-request :crowberto :post 400 "ee/workspace-instance/current"+ {:name "x" :databases {}})]+ (is (re-find #"config\.yml" (:message resp)))))))++(deftest delete-current-rejects-when-locked-test+ (testing "DELETE /ee/workspace-instance/current returns 400 when locked by config"+ (ws.tu/with-workspace-locked-by-config+ (fn []+ (let [resp (mt/user-http-request :crowberto :delete 400 "ee/workspace-instance/current")]+ (is (re-find #"config\.yml" (:message resp))))))))++(deftest delete-current-rejects-when-env-var-set-test+ (testing "DELETE /ee/workspace-instance/current returns 400 when MB_INSTANCE_WORKSPACE is set"+ (mt/with-temp-env-var-value! [:mb-instance-workspace "{\"name\":\"x\",\"databases\":{}}"]+ (let [resp (mt/user-http-request :crowberto :delete 400 "ee/workspace-instance/current")]+ (is (re-find #"config\.yml" (:message resp)))))))++(deftest post-current-rejects-non-superuser-before-lock-test+ (testing "POST /ee/workspace-instance/current returns 403 (not 400) for a non-superuser even when locked"+ (ws.tu/with-workspace-locked-by-config+ (fn []+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :post 403 "ee/workspace-instance/current"+ {:name "x" :databases {}})))))))++(deftest delete-current-rejects-non-superuser-before-lock-test+ (testing "DELETE /ee/workspace-instance/current returns 403 (not 400) for a non-superuser even when locked"+ (ws.tu/with-workspace-locked-by-config+ (fn []+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :delete 403 "ee/workspace-instance/current")))))))++(deftest delete-current-superuser-only-test+ (testing "DELETE /ee/workspace-instance/current requires data analyst or superuser"+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :delete 403 "ee/workspace-instance/current")))))++(deftest delete-current-clears-workspace-test+ (testing "DELETE /ee/workspace-instance/current clears any installed workspace"+ (ws/set-instance-workspace! {:name "to-clear" :databases {}})+ (is (some? (ws/instance-workspace)))+ (mt/user-http-request :crowberto :delete 204 "ee/workspace-instance/current")+ (is (nil? (ws/instance-workspace)))))++(deftest delete-current-drops-table-remappings-test+ (testing "DELETE /ee/workspace-instance/current drops any TableRemapping rows"+ (mt/with-model-cleanup [:model/TableRemapping]+ (ws/set-instance-workspace! {:name "to-clear" :databases {}})+ (t2/insert! :model/TableRemapping {:database_id (mt/id)+ :from_schema "public"+ :from_table_name "orders"+ :to_schema "mb_iso"+ :to_table_name "orders"})+ (is (t2/exists? :model/TableRemapping :database_id (mt/id)))+ (mt/user-http-request :crowberto :delete 204 "ee/workspace-instance/current")+ (is (not (t2/exists? :model/TableRemapping :database_id (mt/id)))))))+(deftest table-remappings-superuser-only-test(testing "GET /ee/workspace-instance/table-remappings requires data analyst or superuser"(is (= "You don't have permissions to do that."diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api/workspace_manager_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api/workspace_manager_test.cljindex e53f349be01..78fd98beac0 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/api/workspace_manager_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api/workspace_manager_test.clj@@ -1,221 +1,114 @@-(ns ^:synchronous metabase-enterprise.workspaces.api.workspace-manager-test+(ns metabase-enterprise.workspaces.api.workspace-manager-test"HTTP smoke tests for the workspace-manager API. Permission rules are exercised at themodel level — see workspace_test.clj and workspace_database_test.clj. These tests justverify routing, request/response shape, and that the model-level permission predicates- are wired into the endpoints (one 403 spot-check is enough).-- A `use-fixtures`-installed `with-redefs` (hence `^:synchronous`) replaces the- real DatabaseProvisioner and InstanceProvisioner with stubs and makes the- `/provision`/`/deprovision` background execution synchronous, so responses- already reflect the final status; individual tests override the provisioners- with their own reify via an inner `with-redefs`."+ are wired into the endpoints (one 403 spot-check is enough)."(:require[clojure.test :refer [deftest is testing use-fixtures]]- [metabase-enterprise.workspaces.provisioning.database :as provisioning.database]- [metabase-enterprise.workspaces.provisioning.instance :as provisioning.instance]+ [metabase-enterprise.workspaces.provisioning :as provisioning][metabase.permissions.test-util :as perms.test-util][metabase.test :as mt][metabase.test.fixtures :as fixtures]- [metabase.util.jvm :as u.jvm][toucan2.core :as t2]))-(defn- stub-database-provisioner []- (reify provisioning.database/DatabaseProvisioner+(use-fixtures :once (fixtures/initialize :db))++(defn- with-premium-feature [f]+ (mt/with-premium-features #{:workspaces}+ (f)))++(use-fixtures :each with-premium-feature)++(defn- stub-provisioner []+ (reify provisioning/Provisioner(details [_ _ _ _]{:schema "mb_iso_stub" :database_details {:user "stub_user" :password "stub_pass"}})(init! [_ _ _ _] nil)(grant! [_ _ _ _ _] nil)(destroy! [_ _ _ _] nil)))-(defn- stub-instance-provisioner []- (reify provisioning.instance/InstanceProvisioner- (create! [_ _workspace _config]- {:id (str (random-uuid)) :url "https://example.com" :status :active})- (fetch [_ instance-id]- {:id instance-id :url "https://example.com" :status :active})- (delete! [_ _instance-id] nil)))--(use-fixtures :once- (fixtures/initialize :db)- (fn [thunk]- (mt/with-premium-features #{:workspaces}- (with-redefs [provisioning.database/database-provisioner (stub-database-provisioner)- provisioning.instance/instance-provisioner (stub-instance-provisioner)- u.jvm/run-in-virtual-thread (fn [work]- (try (work) (catch Throwable _))- nil)]- (thunk)))))-(deftest crud-smoke-test- (testing "create, get, list, delete round-trip — no provisioning involved"+ (testing "create, get, list, delete round-trip"(mt/with-temp [:model/Database {db-id :id} {:engine :postgres:details {}:settings {:database-enable-workspaces true}}](mt/with-model-cleanup [:model/Workspace]- (let [{ws-id :id :as ws} (mt/user-http-request :crowberto :post 200 "ee/workspace-manager/"- {:name "Smoke Test" :database_ids [db-id]})]- (is (=? {:id pos-int?- :name "Smoke Test"- :status "unprovisioned"- :status_details nil- :databases [{:database_id db-id- :status "unprovisioned"}]- :creator some?}- ws)- "POST only creates rows; everything starts :unprovisioned")- (testing "get"- (is (=? {:id ws-id :name "Smoke Test" :status "unprovisioned"}- (mt/user-http-request :crowberto :get 200 (str "ee/workspace-manager/" ws-id)))))- (testing "list"- (is (=? [{:id ws-id}]- (mt/user-http-request :crowberto :get 200 "ee/workspace-manager/"))))- (testing "delete"- (mt/user-http-request :crowberto :delete 204 (str "ee/workspace-manager/" ws-id))- (mt/user-http-request :crowberto :get 404 (str "ee/workspace-manager/" ws-id))))))))--(deftest create-workspace-validation-test- (testing "POST / validates the attached databases without creating anything"- (mt/with-temp [:model/Database {eligible-id :id} {:engine :postgres- :details {}- :settings {:database-enable-workspaces true}}- :model/Table _ {:db_id eligible-id :schema "public" :active true}- :model/Table _ {:db_id eligible-id :schema "analytics" :active true}- :model/Database {ineligible-id :id} {:engine :postgres :details {}}]- (mt/with-model-cleanup [:model/Workspace]- (is (=? {:status "unprovisioned"- :databases [{:database_id eligible-id- :input_schemas ["analytics" "public"]- :status "unprovisioned"}]}- (mt/user-http-request :crowberto :post 200 "ee/workspace-manager/"- {:name "With dbs" :database_ids [eligible-id]}))- "input_schemas are filled from the database's known schemas")- (testing "an ineligible database is rejected"- (mt/user-http-request :crowberto :post 400 "ee/workspace-manager/"- {:name "Nope" :database_ids [ineligible-id]}))- (testing "a missing database is rejected"- (mt/user-http-request :crowberto :post 404 "ee/workspace-manager/"- {:name "Nope" :database_ids [Integer/MAX_VALUE]}))- (testing "at least one database is required"- (mt/user-http-request :crowberto :post 400 "ee/workspace-manager/"- {:name "Nope" :database_ids []}))))))+ (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (let [{ws-id :id :as ws} (mt/user-http-request :crowberto :post 200 "ee/workspace-manager/"+ {:name "Smoke Test" :database_ids [db-id]})]+ (is (=? {:id pos-int?+ :name "Smoke Test"+ :databases [{:database_id db-id}]+ :creator some?}+ ws))+ (testing "get"+ (is (=? {:id ws-id :name "Smoke Test"}+ (mt/user-http-request :crowberto :get 200 (str "ee/workspace-manager/" ws-id)))))+ (testing "list"+ (is (=? [{:id ws-id}]+ (mt/user-http-request :crowberto :get 200 "ee/workspace-manager/"))))+ (testing "delete"+ (mt/user-http-request :crowberto :delete 204 (str "ee/workspace-manager/" ws-id))+ (mt/user-http-request :crowberto :get 404 (str "ee/workspace-manager/" ws-id)))))))))-(deftest create-workspace-target-branch-test- (testing "POST / accepts an optional target_branch and enforces its uniqueness"+(deftest delete-workspace-pending-databases-test+ (testing "DELETE tears down a pending database like any other state"(mt/with-temp [:model/Database {db-id :id} {:engine :postgres:details {}:settings {:database-enable-workspaces true}}](mt/with-model-cleanup [:model/Workspace]- (is (=? {:target_branch "feature-x"}- (mt/user-http-request :crowberto :post 200 "ee/workspace-manager/"- {:name "First" :target_branch "feature-x" :database_ids [db-id]})))- (testing "a duplicate target_branch is rejected"- (mt/user-http-request :crowberto :post 400 "ee/workspace-manager/"- {:name "Second" :target_branch "feature-x" :database_ids [db-id]}))- (testing "target_branch is optional"- (is (=? {:target_branch nil}- (mt/user-http-request :crowberto :post 200 "ee/workspace-manager/"- {:name "Second" :database_ids [db-id]}))))))))--(deftest provision-endpoint-test- (testing "POST /:id/provision provisions the databases and drives the workspace status"- (mt/with-temp [:model/Database {db-id :id} {:engine :postgres- :details {}- :settings {:database-enable-workspaces true}}]- (mt/with-model-cleanup [:model/Workspace]- (let [{ws-id :id} (mt/user-http-request :crowberto :post 200 "ee/workspace-manager/"- {:name "To provision" :database_ids [db-id]})]- (is (=? {:status "database-provisioning"}- (mt/user-http-request :crowberto :post 200- (str "ee/workspace-manager/" ws-id "/provision")))- "the response reflects the just-started run")- (is (=? {:status "provisioned"- :status_details nil- :instance_url "https://example.com"- :databases [{:status "provisioned"- :output_namespace "mb_iso_stub"}]}- (mt/user-http-request :crowberto :get 200 (str "ee/workspace-manager/" ws-id)))- "the stub instance provisioner's url is exposed once the run finishes")- (testing "404 for a missing workspace"- (mt/user-http-request :crowberto :post 404 "ee/workspace-manager/13371337/provision")))))))+ (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (let [{ws-id :id} (mt/user-http-request :crowberto :post 200 "ee/workspace-manager/"+ {:name "Pending" :database_ids [db-id]})+ wsd-id (t2/select-one-pk :model/WorkspaceDatabase :workspace_id ws-id)]+ (t2/update! :model/WorkspaceDatabase {:id wsd-id} {:status :provisioning})+ (mt/user-http-request :crowberto :delete 204 (str "ee/workspace-manager/" ws-id))+ (mt/user-http-request :crowberto :get 404 (str "ee/workspace-manager/" ws-id))))))))-(deftest provision-endpoint-failure-and-retry-test- (testing "a provisioning failure lands in the statuses and a retry finishes the job"+(deftest delete-workspace-teardown-failure-test+ (testing "DELETE returns a 500 with the database's error and keeps the workspace when a teardown fails"(mt/with-temp [:model/Database {db-id :id} {:engine :postgres:details {}:settings {:database-enable-workspaces true}}](mt/with-model-cleanup [:model/Workspace]- (let [{ws-id :id} (mt/user-http-request :crowberto :post 200 "ee/workspace-manager/"- {:name "Flaky" :database_ids [db-id]})- boom (reify provisioning.database/DatabaseProvisioner- (details [_ _ _ _] {:schema "mb_iso_stub" :database_details {}})- (init! [_ _ _ _] (throw (ex-info "boom" {})))+ (let [{ws-id :id} (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (mt/user-http-request :crowberto :post 200 "ee/workspace-manager/"+ {:name "Unreachable" :database_ids [db-id]}))+ boom (reify provisioning/Provisioner+ (details [_ _ _ _] {:schema "mb_iso_stub" :database_details {:user "stub_user"}})+ (init! [_ _ _ _] nil)(grant! [_ _ _ _ _] nil)- (destroy! [_ _ _ _] nil))]- (with-redefs [provisioning.database/database-provisioner boom]- (mt/user-http-request :crowberto :post 200 (str "ee/workspace-manager/" ws-id "/provision"))- (is (=? {:status "database-provisioning-failure"- :status_details "boom"- :databases [{:status "provisioning-failure"- :status_details "boom"}]}- (mt/user-http-request :crowberto :get 200 (str "ee/workspace-manager/" ws-id)))))- (testing "retrying with a healthy warehouse succeeds"- (mt/user-http-request :crowberto :post 200 (str "ee/workspace-manager/" ws-id "/provision"))- (is (=? {:status "provisioned"- :status_details nil}- (mt/user-http-request :crowberto :get 200 (str "ee/workspace-manager/" ws-id))))))))))--(deftest deprovision-endpoint-test- (testing "POST /:id/deprovision tears the databases down and unblocks DELETE"- (mt/with-temp [:model/Database {db-id :id} {:engine :postgres- :details {}- :settings {:database-enable-workspaces true}}]- (mt/with-model-cleanup [:model/Workspace]- (let [{ws-id :id} (mt/user-http-request :crowberto :post 200 "ee/workspace-manager/"- {:name "Round trip" :database_ids [db-id]})]- (mt/user-http-request :crowberto :post 200 (str "ee/workspace-manager/" ws-id "/provision"))- (testing "DELETE refuses while a database is :provisioned"- (mt/user-http-request :crowberto :delete 400 (str "ee/workspace-manager/" ws-id))- (is (t2/exists? :model/Workspace :id ws-id)))- (is (=? {:status "instance-deprovisioning"}- (mt/user-http-request :crowberto :post 200- (str "ee/workspace-manager/" ws-id "/deprovision")))- "the response reflects the just-started run")- (is (=? {:status "unprovisioned"- :status_details nil- :instance_url nil- :databases [{:status "unprovisioned"- :output_namespace ""}]}- (mt/user-http-request :crowberto :get 200 (str "ee/workspace-manager/" ws-id))))- (testing "DELETE works once everything is :unprovisioned"- (mt/user-http-request :crowberto :delete 204 (str "ee/workspace-manager/" ws-id))+ (destroy! [_ _ _ _] (throw (ex-info "Connection refused" {}))))]+ (with-redefs [provisioning/dispatching-provisioner boom]+ (is (=? {:message "Connection refused"}+ (mt/user-http-request :crowberto :delete 500 (str "ee/workspace-manager/" ws-id)))))+ (testing "the workspace survives and a retry deletes it"+ (mt/user-http-request :crowberto :get 200 (str "ee/workspace-manager/" ws-id))+ (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (mt/user-http-request :crowberto :delete 204 (str "ee/workspace-manager/" ws-id)))(mt/user-http-request :crowberto :get 404 (str "ee/workspace-manager/" ws-id))))))))-(deftest provision-refused-while-in-flight-test- (testing "POST /:id/provision and /:id/deprovision 400 while a run is already in flight"- (doseq [in-flight-status [:database-provisioning :instance-provisioning- :instance-deprovisioning :database-deprovisioning]]- (testing (str "workspace status " in-flight-status)- (mt/with-temp [:model/Workspace {ws-id :id} {:name "Busy" :status in-flight-status}]- (mt/user-http-request :crowberto :post 400 (str "ee/workspace-manager/" ws-id "/provision"))- (mt/user-http-request :crowberto :post 400 (str "ee/workspace-manager/" ws-id "/deprovision"))- (mt/user-http-request :crowberto :delete 400 (str "ee/workspace-manager/" ws-id))- (is (= in-flight-status (t2/select-one-fn :status :model/Workspace :id ws-id))- "the in-flight status is untouched"))))- (testing "provision 400s when already :provisioned, deprovision when already :unprovisioned"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "Done" :status :provisioned}]- (mt/user-http-request :crowberto :post 400 (str "ee/workspace-manager/" ws-id "/provision"))- (is (= :provisioned (t2/select-one-fn :status :model/Workspace :id ws-id))))- (mt/with-temp [:model/Workspace {ws-id :id} {:name "Empty" :status :unprovisioned}]- (mt/user-http-request :crowberto :post 400 (str "ee/workspace-manager/" ws-id "/deprovision"))- (is (= :unprovisioned (t2/select-one-fn :status :model/Workspace :id ws-id)))))- (testing "the settled failure statuses stay retryable"- (doseq [settled-status [:database-provisioning-failure :instance-provisioning-failure- :instance-deprovisioning-failure :database-deprovisioning-failure]]- (mt/with-temp [:model/Workspace {ws-id :id} {:name "Settled" :status settled-status}]- (mt/user-http-request :crowberto :post 200 (str "ee/workspace-manager/" ws-id "/provision"))- (is (= :provisioned (t2/select-one-fn :status :model/Workspace :id ws-id)))- (mt/user-http-request :crowberto :post 200 (str "ee/workspace-manager/" ws-id "/deprovision"))- (is (= :unprovisioned (t2/select-one-fn :status :model/Workspace :id ws-id))))))))+(deftest create-workspace-with-database-ids-test+ (testing "POST / attaches and provisions the given databases (each must be eligible) with their schemas"+ (mt/with-temp [:model/Database {eligible-id :id} {:engine :postgres+ :details {}+ :settings {:database-enable-workspaces true}}+ :model/Table _ {:db_id eligible-id :schema "public" :active true}+ :model/Table _ {:db_id eligible-id :schema "analytics" :active true}+ :model/Database {ineligible-id :id} {:engine :postgres :details {}}]+ (mt/with-model-cleanup [:model/Workspace]+ (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (is (=? {:databases [{:database_id eligible-id+ :input_schemas ["analytics" "public"]+ :status "provisioned"}]}+ (mt/user-http-request :crowberto :post 200 "ee/workspace-manager/"+ {:name "With dbs" :database_ids [eligible-id]})))+ (testing "an ineligible database is rejected"+ (mt/user-http-request :crowberto :post 400 "ee/workspace-manager/"+ {:name "Nope" :database_ids [ineligible-id]}))+ (testing "at least one database is required"+ (mt/user-http-request :crowberto :post 400 "ee/workspace-manager/"+ {:name "Nope" :database_ids []})))))))(deftest rename-workspace-test(testing "PUT /:id renames a workspace and returns the updated WorkspaceResponse"@@ -226,6 +119,28 @@(testing "404 for a missing id"(mt/user-http-request :crowberto :put 404 "ee/workspace-manager/13371337" {:name "X"})))))+(deftest download-config-endpoint-test+ (testing "GET /:id/config"+ (mt/with-temp [:model/Database {db-id :id} {:engine :postgres :details {}}+ :model/Workspace {ws-id :id} {:name "Cfg WS"}+ :model/WorkspaceDatabase _ {:workspace_id ws-id+ :database_id db-id+ :database_details {}+ :output_namespace ""+ :input_schemas ["public"]+ :status :provisioned}]+ (testing "returns application/x-yaml with an attachment Content-Disposition"+ (is (=? {:status 200+ :headers {"Content-Type" "application/x-yaml"+ "Content-Disposition" "attachment; filename=\"config.yml\""}}+ (mt/user-http-request-full-response+ :crowberto :get 200 (str "ee/workspace-manager/" ws-id "/config")))))+ (testing "409 when a database is not :provisioned"+ (t2/update! :model/WorkspaceDatabase :workspace_id ws-id {:status :unprovisioned})+ (mt/user-http-request :crowberto :get 409 (str "ee/workspace-manager/" ws-id "/config")))+ (testing "404 for a missing workspace"+ (mt/user-http-request :crowberto :get 404 "ee/workspace-manager/13371337/config")))))+(defmacro ^:private with-data-analyst [& body]`(perms.test-util/with-data-analyst-role! (mt/user->id :rasta) ~@body))@@ -238,6 +153,5 @@(mt/user-http-request :rasta :post 403 "ee/workspace-manager/"{:name "Nope" :database_ids [(mt/id)]})(mt/user-http-request :rasta :put 403 (str "ee/workspace-manager/" ws-id) {:name "Nope"})- (mt/user-http-request :rasta :post 403 (str "ee/workspace-manager/" ws-id "/provision"))- (mt/user-http-request :rasta :post 403 (str "ee/workspace-manager/" ws-id "/deprovision"))+ (mt/user-http-request :rasta :get 403 (str "ee/workspace-manager/" ws-id "/config"))(mt/user-http-request :rasta :delete 403 (str "ee/workspace-manager/" ws-id))))))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/config_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/config_test.cljindex 157c8588759..885e650729a 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/config_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/config_test.clj@@ -5,18 +5,10 @@[metabase-enterprise.workspaces.test-util :as workspaces.tu][metabase.test :as mt][metabase.test.fixtures :as fixtures]- [metabase.util :as u]- [toucan2.core :as t2]))+ [metabase.util :as u]))(use-fixtures :once (fixtures/initialize :db))-(defn- build-config- "Fetch + hydrate the workspace the way production callers do, then build."- [ws-id]- (-> (t2/select-one :model/Workspace :id ws-id)- (t2/hydrate :databases)- config/build-workspace-config))-(deftest build-workspace-config-happy-path-test(testing "build-workspace-config returns a {:version 1 :config {...}} structure matching config.yml"(mt/with-temp [:model/Database {db-id :id}@@ -35,7 +27,7 @@:output_namespace "mb_isolation_github":input_schemas ["raw_github"]:status :provisioned}]- (let [cfg (build-config ws-id)]+ (let [cfg (config/build-workspace-config ws-id)](testing "outer shape matches config.yml (version + config block)"(is (= 1 (:version cfg)))(is (= #{:databases :workspace} (set (keys (:config cfg))))))@@ -78,7 +70,7 @@:output_namespace "ws_alice":input_schemas ["dbo"]:status :provisioned}]- (let [cfg (build-config ws-id)]+ (let [cfg (config/build-workspace-config ws-id)](is (= {"MSSQL DW"{:input_schemas ["dbo"]:output {:db "AnalyticsDB" :schema "ws_alice"}}}@@ -97,7 +89,7 @@:output_namespace "out":input_schemas ["schema_a" "schema_b" "schema_c"]:status :provisioned}]- (let [cfg (build-config ws-id)]+ (let [cfg (config/build-workspace-config ws-id)](is (= "schema_a,schema_b,schema_c"(-> cfg :config :databases first :details :schema-filters-patterns)))))))@@ -123,7 +115,7 @@:output_namespace "ws_alice":input_schemas ["core" "warehouse"]:status :provisioned}]- (let [details (-> (build-config ws-id) :config :databases first :details)]+ (let [details (-> (config/build-workspace-config ws-id) :config :databases first :details)](is (= "inclusion" (:dataset-filters-type details)))(is (= "core,warehouse" (:dataset-filters-patterns details)))(is (some? (:service-account-json details))@@ -143,13 +135,13 @@(is (thrown-with-msg?Exception#"not :provisioned"- (build-config ws-id))))))+ (config/build-workspace-config ws-id))))))(deftest build-workspace-config-empty-workspace-test(testing "A workspace with no databases still produces the version+config outer shape"(mt/with-temp [:model/Workspace {ws-id :id} {:name "Empty":creator_id (mt/user->id :crowberto)}]- (let [cfg (build-config ws-id)]+ (let [cfg (config/build-workspace-config ws-id)](is (= 1 (:version cfg)))(is (= "Empty" (-> cfg :config :workspace :name)))(is (empty? (remove (some-fn :is_stub :is_sample) (-> cfg :config :databases)))@@ -174,7 +166,7 @@:output_namespace "":input_schemas ["public"]:status :provisioned}]- (let [cfg-dbs (-> (build-config ws-id) :config :databases)+ (let [cfg-dbs (-> (config/build-workspace-config ws-id) :config :databases)by-name (into {} (map (juxt :name identity)) cfg-dbs)](testing "workspace's own database is present and is NOT a stub"(is (contains? by-name "stub-test-ws-db"))@@ -196,6 +188,11 @@(is (not (contains? by-name "stub-test-audit")))(is (not (contains? by-name "stub-test-target"))))))))+(deftest build-workspace-config-missing-workspace-returns-nil-test+ (testing "A missing workspace returns nil"+ (mt/with-model-cleanup [:model/Workspace]+ (is (nil? (config/build-workspace-config Integer/MAX_VALUE))))))+(deftest build-workspace-config-forces-let-user-control-scheduling-false-test(testing "Non-stub database entries emit :let-user-control-scheduling false even if the sourcedatabase had it true. The YAML carries :details only — not the schedule cron columns —@@ -216,7 +213,7 @@:output_namespace "ws_alice":input_schemas ["public"]:status :provisioned}]- (let [own-db (->> (build-config ws-id)+ (let [own-db (->> (config/build-workspace-config ws-id):config:databases(remove (some-fn :is_stub :is_sample))@@ -239,7 +236,7 @@;; Select the MySQL entry explicitly rather than relying on `first` -- `:databases` also carries;; stub/sample entries (e.g. a pre-existing Sample Database), so its ordering isn't something this;; test should depend on.- (let [details (->> (build-config ws-id) :config :databases+ (let [details (->> (config/build-workspace-config ws-id) :config :databases(u/seek #(= :mysql (:engine %))):details)](is (nil? (:schema-filters-type details)))@@ -258,7 +255,7 @@:is_sample true}:model/Workspace {ws-id :id} {:name "sample-emit-ws":creator_id (mt/user->id :crowberto)}]- (let [cfg-dbs (-> (build-config ws-id) :config :databases)+ (let [cfg-dbs (-> (config/build-workspace-config ws-id) :config :databases)samples (filter :is_sample cfg-dbs)](testing "exactly one sample entry is emitted"(is (= 1 (count samples))))@@ -270,43 +267,3 @@(first samples))))(testing "sample entry is NOT also marked as a stub"(is (not (:is_stub (first samples)))))))))--(deftest build-workspace-config-remote-sync-settings-test- (testing "when remote sync is enabled, :config carries read-write remote-sync settings for the child instance"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "github"- :creator_id (mt/user->id :crowberto)}]- (mt/with-temporary-setting-values [remote-sync-url "https://git.example.com/test.git"- remote-sync-branch "main"- remote-sync-token "s3cr3t-token"]- (is (= {:remote-sync-url "https://git.example.com/test.git"- :remote-sync-branch "main"- :remote-sync-token "s3cr3t-token"- :remote-sync-type :read-write}- (-> (build-config ws-id) :config :settings))))- (testing "and no :settings section when remote sync is disabled"- (mt/with-temporary-setting-values [remote-sync-url nil]- (is (not (contains? (-> (build-config ws-id) :config) :settings)))))))- (testing "the workspace's target_branch overrides the instance's remote-sync branch"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "github"- :creator_id (mt/user->id :crowberto)- :target_branch "workspace-branch"}]- (mt/with-temporary-setting-values [remote-sync-url "https://git.example.com/test.git"- remote-sync-branch "main"- remote-sync-token "s3cr3t-token"]- (is (= "workspace-branch"- (-> (build-config ws-id) :config :settings :remote-sync-branch)))))))--(deftest build-workspace-config-creator-superuser-test- (testing "when workspace-instance-user-password is set, :config carries the creator as a superuser"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "github"- :creator_id (mt/user->id :crowberto)}]- (mt/with-temporary-setting-values [workspace-instance-user-password "sUp3r-s3cret"]- (is (= [{:first_name "Crowberto"- :last_name "Corv"- :email "[email protected]"- :password "sUp3r-s3cret"- :is_superuser true}]- (-> (build-config ws-id) :config :users))))- (testing "and no :users section when the password is not set"- (mt/with-temporary-setting-values [workspace-instance-user-password nil]- (is (not (contains? (-> (build-config ws-id) :config) :users))))))))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/core_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/core_test.cljnew file mode 100644index 00000000000..a67e18955d1--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/workspaces/core_test.clj@@ -0,0 +1,316 @@+(ns metabase-enterprise.workspaces.core-test+ "Tests for the workspace programmatic API and lifecycle rules."+ (:require+ [clojure.string :as str]+ [clojure.test :refer [deftest is testing use-fixtures]]+ [metabase-enterprise.workspaces.core :as ws]+ [metabase-enterprise.workspaces.provisioning :as provisioning]+ [metabase-enterprise.workspaces.test-util :as workspaces.tu]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2])+ (:import+ (clojure.lang ExceptionInfo)))++(set! *warn-on-reflection* true)++(use-fixtures :once (fixtures/initialize :db))++(defn- with-premium-feature [f]+ (mt/with-premium-features #{:workspaces}+ (f)))++(use-fixtures :each with-premium-feature)++;;; Stub provisioner: records calls but doesn't hit real drivers.+(defn- stub-provisioner []+ (reify provisioning/Provisioner+ (details [_ _ _ _]+ {:schema "mb_iso_stub" :database_details {:user "stub_user" :password "stub_pass"}})+ (init! [_ _ _ _] nil)+ (grant! [_ _ _ _ _] nil)+ (destroy! [_ _ _ _] nil)))++;;; ----------------------------------------------- CRUD -------------------------------------------------------++(deftest create-workspace-test+ (testing "create with name only (no databases)"+ (mt/with-model-cleanup [:model/Workspace]+ (let [ws (ws/create-workspace! {:name "Test WS"+ :creator_id (mt/user->id :crowberto)})]+ (is (= "Test WS" (:name ws)))+ (is (empty? (:databases ws)))))))++(deftest get-and-list-test+ (testing "get returns nil for non-existent"+ (is (nil? (ws/get-workspace 999999))))+ (testing "get returns hydrated workspace"+ (mt/with-model-cleanup [:model/Workspace]+ (let [ws (ws/create-workspace! {:name "Get Test"+ :creator_id (mt/user->id :crowberto)})+ fetched (ws/get-workspace (:id ws))]+ (is (some? (:creator fetched)))+ (is (= [] (:databases fetched))))))+ (testing "list returns all"+ (mt/with-model-cleanup [:model/Workspace]+ (ws/create-workspace! {:name "A" :creator_id (mt/user->id :crowberto)})+ (ws/create-workspace! {:name "B" :creator_id (mt/user->id :crowberto)})+ (is (>= (count (ws/list-workspaces)) 2)))))++(deftest create-workspace-provisions-databases-test+ (testing "create-workspace! attaches and provisions the given databases"+ (mt/with-temp [:model/Database {db-id :id} {:engine :postgres+ :details {}+ :settings {:database-enable-workspaces true}}+ :model/Table _ {:db_id db-id :schema "public" :active true}+ :model/Database {ineligible-id :id} {:engine :postgres :details {}}]+ (mt/with-model-cleanup [:model/Workspace]+ (testing "an ineligible database is rejected"+ (is (thrown-with-msg? ExceptionInfo #"Workspaces are not enabled for this database"+ (ws/create-workspace! {:name "Nope"+ :creator_id (mt/user->id :crowberto)+ :database_ids [ineligible-id]}))))+ (testing "a missing database is rejected"+ (is (thrown-with-msg? ExceptionInfo #"Database not found"+ (ws/create-workspace! {:name "Nope"+ :creator_id (mt/user->id :crowberto)+ :database_ids [Integer/MAX_VALUE]}))))+ (testing "provisioning failure cleans up: databases torn down, workspace and rows deleted"+ (let [failing-provisioner (reify provisioning/Provisioner+ (details [_ _ _ _] {:schema "mb_iso_stub" :database_details {:user "stub_user"}})+ (init! [_ _ _ _] (throw (ex-info "boom" {})))+ (grant! [_ _ _ _ _] nil)+ (destroy! [_ _ _ _] nil))]+ (with-redefs [provisioning/dispatching-provisioner failing-provisioner]+ (is (thrown-with-msg? ExceptionInfo #"boom"+ (ws/create-workspace! {:name "Boom"+ :creator_id (mt/user->id :crowberto)+ :database_ids [db-id]}))))+ (is (not (t2/exists? :model/Workspace :name "Boom"))+ "the failed create must not leave a Workspace row behind")+ (is (not (t2/exists? :model/WorkspaceDatabase :database_id db-id))+ "the failed create must not leave WorkspaceDatabase rows behind")))+ (testing "provisioning failure whose cleanup also fails keeps the workspace and returns it for retry"+ (let [wedged (reify provisioning/Provisioner+ (details [_ _ _ _] {:schema "mb_iso_stub" :database_details {:user "stub_user"}})+ (init! [_ _ _ _] (throw (ex-info "boom" {})))+ (grant! [_ _ _ _ _] nil)+ (destroy! [_ _ _ _] (throw (ex-info "warehouse down" {}))))]+ (with-redefs [provisioning/dispatching-provisioner wedged]+ (is (=? {:name "Wedged"+ :databases [{:status :unprovisioned}]}+ (ws/create-workspace! {:name "Wedged"+ :creator_id (mt/user->id :crowberto)+ :database_ids [db-id]}))+ "the created workspace is returned even though provisioning failed and cleanup could not finish"))+ (let [ws (t2/select-one :model/Workspace :name "Wedged")]+ (is (some? ws)+ "the workspace must be kept while any database row remains")+ (is (=? [{:status :unprovisioned}]+ (t2/select :model/WorkspaceDatabase :workspace_id (:id ws)))+ "the row whose teardown failed is kept, forced :unprovisioned")+ ;; retry the teardown via delete once the warehouse is back+ (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (is (nil? (ws/delete-workspace! (:id ws)))))+ (is (nil? (ws/get-workspace (:id ws)))))))+ (testing "success: the attached database comes back :provisioned"+ (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (let [ws (ws/create-workspace! {:name "Provisioned"+ :creator_id (mt/user->id :crowberto)+ :database_ids [db-id]})]+ (is (=? [{:database_id db-id+ :input_schemas ["public"]+ :status :provisioned}]+ (:databases ws))))))))))++(defn- add-database!+ "Test helper: insert a WorkspaceDatabase row for `workspace-id` and provision it+ (blocking). Replaces the removed production add-database! for test setup."+ [workspace-id database-id input-schemas]+ (t2/with-transaction [_conn]+ (let [wsd-id (t2/insert-returning-pk! :model/WorkspaceDatabase+ {:workspace_id workspace-id+ :database_id database-id+ :input_schemas input-schemas+ :database_details {}+ :output_namespace ""})]+ (provisioning/provision-single! wsd-id))))++;;; ----------------------------------------- Delete Workspace ------------------------------------------------++(deftest delete-workspace-test+ (testing "delete deprovisions all databases first"+ (mt/with-model-cleanup [:model/Workspace]+ (let [ws (ws/create-workspace! {:name "Delete WS" :creator_id (mt/user->id :crowberto)})]+ (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (add-database! (:id ws) (mt/id) ["PUBLIC"]))+ (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (ws/delete-workspace! (:id ws)))+ (is (nil? (ws/get-workspace (:id ws)))))))+ (testing "delete workspace with no databases"+ (mt/with-model-cleanup [:model/Workspace]+ (let [ws (ws/create-workspace! {:name "Empty WS" :creator_id (mt/user->id :crowberto)})]+ (is (nil? (ws/delete-workspace! (:id ws))))+ (is (nil? (ws/get-workspace (:id ws))))))))++(deftest delete-workspace-with-pending-status-test+ (testing "a workspace database still :provisioning/:deprovisioning is torn down like any other state"+ (doseq [pending-status [:provisioning :deprovisioning]]+ (testing (str "status " pending-status)+ (mt/with-model-cleanup [:model/Workspace]+ (let [ws (ws/create-workspace! {:name (str "Pending " (name pending-status))+ :creator_id (mt/user->id :crowberto)})+ _ (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (add-database! (:id ws) (mt/id) ["PUBLIC"]))+ wsd-id (t2/select-one-pk :model/WorkspaceDatabase :workspace_id (:id ws))]+ ;; force the row into the pending state. :provisioning rows that crashed+ ;; never stored their iso details, so clear them too.+ (t2/update! :model/WorkspaceDatabase {:id wsd-id}+ (cond-> {:status pending-status}+ (= pending-status :provisioning)+ (assoc :output_namespace "" :database_details {})))+ (let [destroyed? (atom false)+ recording (reify provisioning/Provisioner+ (details [_ _ _ _] {:schema "mb_iso_stub" :database_details {:user "stub_user"}})+ (init! [_ _ _ _] nil)+ (grant! [_ _ _ _ _] nil)+ (destroy! [_ _ _ _] (reset! destroyed? true) nil))]+ (with-redefs [provisioning/dispatching-provisioner recording]+ (is (nil? (ws/delete-workspace! (:id ws)))))+ (is (true? @destroyed?)+ "pending rows get a real warehouse teardown, not an app-DB-only removal")+ (is (nil? (ws/get-workspace (:id ws))))+ (is (not (t2/exists? :model/WorkspaceDatabase :id wsd-id))))))))))++(deftest delete-workspace-teardown-failure-test+ (testing "when teardown fails, nothing is deleted: the row and the workspace are kept for retry"+ (mt/with-model-cleanup [:model/Workspace]+ (let [ws (ws/create-workspace! {:name "Unreachable WS" :creator_id (mt/user->id :crowberto)})+ _ (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (add-database! (:id ws) (mt/id) ["PUBLIC"]))+ wsd-id (t2/select-one-pk :model/WorkspaceDatabase :workspace_id (:id ws))+ boom (reify provisioning/Provisioner+ (details [_ _ _ _] {:schema "mb_iso_stub" :database_details {:user "stub_user"}})+ (init! [_ _ _ _] nil)+ (grant! [_ _ _ _ _] nil)+ (destroy! [_ _ _ _] (throw (ex-info "Connection refused" {}))))]+ (with-redefs [provisioning/dispatching-provisioner boom]+ (is (thrown-with-msg? ExceptionInfo #"Connection refused"+ (ws/delete-workspace! (:id ws)))+ "the combined teardown failure is thrown, carrying what the database returned"))+ (is (some? (ws/get-workspace (:id ws)))+ "the workspace must be kept when any teardown fails")+ (is (=? {:status :unprovisioned}+ (t2/select-one :model/WorkspaceDatabase :id wsd-id))+ "the failed row is kept, forced :unprovisioned")+ (testing "retrying the delete once the warehouse is reachable finishes the job"+ (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (is (nil? (ws/delete-workspace! (:id ws)))))+ (is (nil? (ws/get-workspace (:id ws))))+ (is (not (t2/exists? :model/WorkspaceDatabase :id wsd-id))))))))++(deftest delete-workspace-combines-teardown-failures-test+ (testing "every database gets its teardown attempt and all failures come back as one combined exception"+ (mt/with-temp [:model/Database {db2-id :id} {:engine :postgres+ :details {}+ :settings {:database-enable-workspaces true}}]+ (mt/with-model-cleanup [:model/Workspace]+ (let [ws (ws/create-workspace! {:name "Multi fail" :creator_id (mt/user->id :crowberto)})+ _ (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (add-database! (:id ws) (mt/id) ["PUBLIC"])+ (add-database! (:id ws) db2-id ["public"]))+ boom (reify provisioning/Provisioner+ (details [_ _ _ _] {:schema "mb_iso_stub" :database_details {:user "stub_user"}})+ (init! [_ _ _ _] nil)+ (grant! [_ _ _ _ _] nil)+ (destroy! [_ _ db _] (throw (ex-info (str "down: " (:id db)) {}))))+ e (with-redefs [provisioning/dispatching-provisioner boom]+ (try+ (ws/delete-workspace! (:id ws))+ nil+ (catch Throwable t t)))]+ (is (some? e))+ (is (= #{(str "down: " (mt/id)) (str "down: " db2-id)}+ (set (str/split (ex-message e) #"; ")))+ "the message joins what each database returned")+ (is (some? (ex-cause e))+ "the first failure is the cause")+ (is (= 1 (count (.getSuppressed ^Throwable e)))+ "the remaining failures are suppressed")+ (is (= 2 (t2/count :model/WorkspaceDatabase :workspace_id (:id ws)))+ "both rows are kept for retry")+ (with-redefs [provisioning/dispatching-provisioner (stub-provisioner)]+ (is (nil? (ws/delete-workspace! (:id ws))))))))))++;;; -------------------------------------------- Remappings ----------------------------------------------------++(deftest list-remappings-test+ (mt/with-model-cleanup [:model/TableRemapping]+ (t2/insert! :model/TableRemapping {:database_id (mt/id)+ :from_schema "public"+ :from_table_name "orders"+ :to_schema "mb_iso"+ :to_table_name "orders"})+ (let [remappings (ws/list-remappings)]+ (is (some #(= "orders" (:from_table_name %)) remappings)))))++;;; ----------------------------------------- workspace-instance lock ------------------------------------------++(defn- lock-atom+ "Reach through the private var to the lock atom."+ []+ @#'ws/locked-by-config?*)++(deftest workspace-locked-by-config?-baseline-test+ (testing "default state (no atom flip, no env-var) is unlocked"+ (let [a (lock-atom)+ prior @a]+ (try+ (reset! a false)+ (is (false? (ws/workspace-locked-by-config?)))+ (finally (reset! a prior))))))++(deftest workspace-locked-by-config?-atom-flipped-test+ (testing "after mark-locked-by-config!, the predicate returns true"+ (workspaces.tu/with-workspace-locked-by-config+ (fn [] (is (true? (ws/workspace-locked-by-config?)))))))++(deftest mark-locked-by-config!-flips-the-atom-test+ (testing "mark-locked-by-config! flips the atom to true"+ (let [a (lock-atom)+ prior @a]+ (try+ (reset! a false)+ (is (false? (ws/workspace-locked-by-config?)))+ (ws/mark-locked-by-config!)+ (is (true? @a))+ (is (true? (ws/workspace-locked-by-config?)))+ (finally (reset! a prior))))))++(deftest env-var-presence-locks-test+ (testing "MB_INSTANCE_WORKSPACE set => predicate true regardless of atom"+ (let [a (lock-atom)+ prior @a]+ (try+ (reset! a false)+ (mt/with-temp-env-var-value! [mb-instance-workspace "{\"name\":\"x\",\"databases\":{}}"]+ (is (true? (ws/workspace-locked-by-config?))))+ (testing "outside the binding, atom alone determines the lock"+ (is (false? (ws/workspace-locked-by-config?))))+ (finally (reset! a prior))))))++(deftest env-var-empty-string-does-not-lock-test+ (testing "MB_INSTANCE_WORKSPACE set to empty string does NOT lock (env-var-value treats blank as unset)"+ (let [a (lock-atom)+ prior @a]+ (try+ (reset! a false)+ (mt/with-temp-env-var-value! [mb-instance-workspace ""]+ (is (false? (ws/workspace-locked-by-config?))))+ (finally (reset! a prior))))))++(deftest env-var-and-atom-both-lock-test+ (testing "both sources set => still true; OR semantics"+ (mt/with-temp-env-var-value! [mb-instance-workspace "{\"name\":\"x\",\"databases\":{}}"]+ (workspaces.tu/with-workspace-locked-by-config+ (fn [] (is (true? (ws/workspace-locked-by-config?))))))))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/e2e_test.cljindex 0028881e46f..be3e048119c 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/e2e_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/e2e_test.clj@@ -29,10 +29,8 @@[clojure.test :refer [deftest is testing]][metabase-enterprise.advanced-config.file :as advanced-config.file][metabase-enterprise.workspaces.config :as ws.config]- [metabase-enterprise.workspaces.instance :as ws.instance]- [metabase-enterprise.workspaces.models.workspace :as workspace]+ [metabase-enterprise.workspaces.core :as ws][metabase-enterprise.workspaces.provisioning :as provisioning]- [metabase-enterprise.workspaces.provisioning.database :as provisioning.database][metabase-enterprise.workspaces.table-remapping :as ws.table-remapping][metabase.driver :as driver][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]@@ -100,28 +98,13 @@(blocking). Replaces the removed production add-database! for test setup."[workspace-id database-id input-schemas](t2/with-transaction [_conn]- (let [wsd (t2/insert-returning-instance! :model/WorkspaceDatabase- {:workspace_id workspace-id- :database_id database-id- :input_schemas input-schemas- :database_details {}- :output_namespace ""})]- (provisioning.database/provision-database! wsd))))--(defn- cleanup-workspace!- "Best-effort test cleanup: deprovision every database (destroys the warehouse- isolation; failures land in the statuses, no throw), then delete the- workspace. When the deprovision didn't fully succeed the delete refuses, so- fall back to force-clearing the rows — otherwise the `mt/with-temp Database`- cleanup can't complete."- [ws-id]- (when-let [ws (t2/select-one :model/Workspace :id ws-id)]- (try- (provisioning/deprovision-workspace! ws)- (catch Throwable t- (log/warnf t "deprovision-workspace! failed during cleanup for workspace %d" ws-id)))- (t2/delete! :model/WorkspaceDatabase :workspace_id ws-id)- (t2/delete! :model/Workspace :id ws-id)))+ (let [wsd-id (t2/insert-returning-pk! :model/WorkspaceDatabase+ {:workspace_id workspace-id+ :database_id database-id+ :input_schemas input-schemas+ :database_details {}+ :output_namespace ""})]+ (provisioning/provision-single! wsd-id))));;; -------------------- driver-branched helpers --------------------;;;@@ -365,19 +348,19 @@(mt/with-temp [:model/Database ws-db {:engine admin-driver:details admin-details:name (str "ws-e2e-" run-id)}]- (let [{ws-id :id} (workspace/create-workspace! {:name (str "ws-e2e-" run-id)- :creator_id (mt/user->id :crowberto)})]+ (let [{ws-id :id} (ws/create-workspace! {:name (str "ws-e2e-" run-id)+ :creator_id (mt/user->id :crowberto)})](try;; --- Stage 1: provision via the workspace provisioning entrypoint.;; Drives the same `init-workspace-isolation!` + `grant-workspace-read-access!`- ;; multimethods, but through `provisioning.database/provision-database!`, which- ;; writes the resulting `:database_details` and `:output_namespace` back to the+ ;; multimethods, but through `provisioning/provision-single!`, which writes+ ;; the resulting `:database_details` and `:output_namespace` back to the;; `WorkspaceDatabase` row — the inputs `build-workspace-config` reads.(add-database! ws-id (:id ws-db)[(workspace-input-schema admin-driver admin-details main-schema)]);; Sanity check: provisioning must have populated :output_namespace and flipped;; status to :provisioned. If empty, the workspace driver impl is broken.- (let [wsd (t2/select-one :model/WorkspaceDatabase :workspace_id ws-id)]+ (let [wsd (-> (ws/get-workspace ws-id) :databases first)](is (= :provisioned (:status wsd))"WorkspaceDatabase provisioning did not reach :provisioned status")(is (and (string? (:output_namespace wsd))@@ -390,13 +373,12 @@;; `:engine :postgres` (keyword), but the `:databases` section spec requires;; a string. [[yaml/parse-string]] or [[yaml/generate-string]] collapses keyword;; values to strings, matching the on-disk wire format.- (let [yaml-str (-> (t2/select-one :model/Workspace :id ws-id)- (t2/hydrate :databases)+ (let [yaml-str (-> ws-idws.config/build-workspace-configws.config/config->yaml);; Read the provisioned isolation schema name back from the WSD row;- ;; `provision-database!` derives it from the WSD id, not the workspace id.- wsd (t2/select-one :model/WorkspaceDatabase :workspace_id ws-id)+ ;; `provision-single!` derives it from the WSD id, not the workspace id.+ wsd (-> (ws/get-workspace ws-id) :databases first)isolation-schema (:output_namespace wsd)];;+---------------------------+;;| Parent above |@@ -416,7 +398,7 @@(advanced-config.file/initialize! (yaml/parse-string yaml-str));; Diagnostic: the loader should have written the workspace-instance;; setting and rewritten the Database row's :details to workspace-user creds.- (is (ws.instance/workspace-mode?)+ (is (ws/workspace-mode?)"loader did not put the instance into workspace-mode (atom not populated)");; The Database row's `:details` was just rewritten by the loader. Re-read;; it so `mt/with-db` and the connection pool see the workspace-user creds.@@ -683,15 +665,25 @@;; `:databases` initializer rewrote the `Database.details` to the workspace;; user's creds — but `destroy-workspace-isolation!` needs admin privileges;; (DROP DATABASE / DROP USER on MySQL, etc), so restore admin details first.- ;; `cleanup-workspace!` then deprovisions every database (any state, calls- ;; `destroy-workspace-isolation!`) before deleting the workspace, safe whether+ ;; `delete-workspace!` then tears down every database (any state, calls+ ;; `destroy-workspace-isolation!`) before deleting the row, safe whether;; provision succeeded fully or partially.- (ws.instance/clear-instance-workspace!)+ (ws/clear-instance-workspace!)(t2/update! :model/Database (:id ws-db) {:details admin-details})- ;; See [[cleanup-workspace!]]: deprovision (destroys the warehouse- ;; isolation), delete, force-clear on failure. Real fix for the+ ;; Cleanup tries [[ws/delete-workspace!]] first. A failed warehouse+ ;; teardown throws and keeps the workspace + rows for retry. For test+ ;; cleanup we don't retry: fall back to force-clearing the rows so+ ;; `mt/with-temp Database` cleanup can complete. Real fix for the;; Redshift destroy fragility is tracked separately.- (cleanup-workspace! ws-id)))))+ (let [deleted? (try (ws/delete-workspace! ws-id)+ true+ (catch Throwable t+ (log/warnf t "delete-workspace! left workspace %d behind; force-clearing WSD"+ ws-id)+ false))]+ (when-not deleted?+ (t2/delete! :model/WorkspaceDatabase :workspace_id ws-id)+ (t2/delete! :model/Workspace :id ws-id)))))))(finally(try (drop-canonical-schema! admin-driver admin-spec admin-details main-schema src-name output-table-name)(catch Throwable _ nil))))))))))))@@ -739,14 +731,12 @@(mt/with-temp [:model/Database ws-db {:engine admin-driver:details admin-details:name (str "ws-native-repro-" run-id)}]- (let [{ws-id :id} (workspace/create-workspace! {:name (str "ws-native-repro-" run-id)- :creator_id (mt/user->id :crowberto)})]+ (let [{ws-id :id} (ws/create-workspace! {:name (str "ws-native-repro-" run-id)+ :creator_id (mt/user->id :crowberto)})](try(add-database! ws-id (:id ws-db)[(workspace-input-schema admin-driver admin-details main-schema)])- (let [cfg-map (-> (t2/select-one :model/Workspace :id ws-id)- (t2/hydrate :databases)- ws.config/build-workspace-config)+ (let [cfg-map (ws.config/build-workspace-config ws-id)yaml-str (ws.config/config->yaml cfg-map)reparsed (yaml/parse-string yaml-str)](advanced-config.file/initialize! reparsed)@@ -824,9 +814,11 @@(is (= [{:n 3}] (vec rows))"B's iso output reflects rewriter routing B's SELECT to A's iso table"))))))))))(finally- (ws.instance/clear-instance-workspace!)+ (ws/clear-instance-workspace!)(t2/update! :model/Database (:id ws-db) {:details admin-details})- (cleanup-workspace! ws-id)))))+ (try (ws/delete-workspace! ws-id)+ (catch Throwable t+ (log/warn t "delete-workspace! failed during native-transform repro cleanup")))))))(finally(try (drop-canonical-schema! admin-driver admin-spec admin-details main-schema src-name tgt-a-name)(catch Throwable _ nil))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/models/workspace_database_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/models/workspace_database_test.cljindex b068157f3dd..0e868083cdc 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/models/workspace_database_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/models/workspace_database_test.clj@@ -4,8 +4,8 @@Background: `workspace_database.database_id -> metabase_database.id` used to beON DELETE CASCADE, which silently removed the row when the parent Database was- deleted — orphaning any `:provisioned` warehouse schemas/users because the- app-side delete guards were never consulted. We switched the FK to+ deleted — orphaning any `:provisioned` warehouse schemas/users because the 409+ guard in `delete-workspace!` was never consulted. We switched the FK toRESTRICT, which requires the app-side to reconcile rows explicitly: refuse thedelete if any row is `:provisioned`, otherwise delete the `:unprovisioned`rows so the FK is satisfied."diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/models/workspace_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/models/workspace_test.cljindex 0f6fbef01cc..aff091ab486 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/models/workspace_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/models/workspace_test.clj@@ -17,11 +17,9 @@overrides)))(defn- create-ws!- "Test helper: supply a default :creator_id so call sites stay terse, and- hydrate :databases the way the API does."+ "Test helper: supply a default :creator_id so call sites stay terse."[params]- (-> (workspace/create-workspace! (merge {:creator_id (mt/user->id :crowberto)} params))- (t2/hydrate :databases)))+ (workspace/create-workspace! (merge {:creator_id (mt/user->id :crowberto)} params)))(deftest create-workspace-minimal-test(testing "create-workspace! creates a Workspace with no databases"@@ -51,6 +49,68 @@(is (= ["public" "analytics"](:input_schemas first-db))))))))))+(deftest get-workspace-test+ (testing "get-workspace returns a hydrated workspace"+ (mt/with-model-cleanup [:model/Workspace]+ (let [{id :id} (create-ws!+ {:name "Fetch Me" :databases [(ws-db-attrs)]})+ fetched (workspace/get-workspace id)]+ (is (= "Fetch Me" (:name fetched)))+ (is (= 1 (count (:databases fetched))))+ (is (= (mt/id) (:database_id (first (:databases fetched))))))))+ (testing "get-workspace returns nil for a missing id"+ (is (nil? (workspace/get-workspace Integer/MAX_VALUE)))))++(deftest list-workspaces-test+ (testing "list-workspaces returns all workspaces with their databases hydrated"+ (mt/with-model-cleanup [:model/Workspace]+ (let [{id-a :id} (create-ws!+ {:name "A" :databases [(ws-db-attrs)]})+ {id-b :id} (create-ws!+ {:name "B" :databases []})+ results (workspace/list-workspaces)+ by-id (into {} (map (juxt :id identity)) results)]+ (is (contains? by-id id-a))+ (is (contains? by-id id-b))+ (is (= 1 (count (:databases (get by-id id-a)))))+ (is (= [] (:databases (get by-id id-b))))))))++(deftest update-workspace-replaces-databases-test+ (testing "update-workspace! replaces the set of workspace_databases"+ (mt/with-model-cleanup [:model/Workspace]+ (mt/with-temp [:model/Database {db2-id :id} {:engine :h2 :details {}}]+ (let [{id :id} (create-ws!+ {:name "Before"+ :databases [(ws-db-attrs {:output_namespace "keep_out"+ :input_schemas ["keep"]})+ (ws-db-attrs {:database_id db2-id+ :output_namespace "drop_out"+ :input_schemas ["drop"]})]})+ updated (workspace/update-workspace!+ id+ {:name "After"+ :databases [(ws-db-attrs {:output_namespace "keep_out"+ :input_schemas ["keep"]})+ (ws-db-attrs {:database_id db2-id+ :output_namespace "new_out"+ :input_schemas ["new"]})]})]+ (is (= "After" (:name updated)))+ (is (= #{"keep_out" "new_out"}+ (into #{} (map :output_namespace) (:databases updated))))+ (testing "the removed workspace_database is gone from the database"+ (is (not (t2/exists? :model/WorkspaceDatabase+ :workspace_id id+ :output_namespace "drop_out")))))))))++(deftest update-workspace-can-clear-databases-test+ (testing "update-workspace! with empty :databases removes all children"+ (mt/with-model-cleanup [:model/Workspace]+ (let [{id :id} (create-ws!+ {:name "To Clear" :databases [(ws-db-attrs)]})+ updated (workspace/update-workspace! id {:name "Cleared" :databases []})]+ (is (= [] (:databases updated)))+ (is (not (t2/exists? :model/WorkspaceDatabase :workspace_id id)))))))+(deftest cascade-delete-workspace-test(testing "Deleting a Workspace cascades to its workspace_database rows"(mt/with-model-cleanup [:model/Workspace]@@ -83,16 +143,77 @@:from [:workspace_database]:where [:= :workspace_id (:id created)]})))))))))-(deftest status-details-encrypted-round-trip-test- (testing "workspace.status_details and workspace_database.status_details round-trip transparently"+(deftest status-can-be-updated-test+ (testing "update-workspace! can change the :status of a workspace_database"(mt/with-model-cleanup [:model/Workspace]- (let [{id :id} (create-ws! {:name "Statuses" :databases [(ws-db-attrs)]})]- (t2/update! :model/Workspace id {:status :database-provisioning-failure, :status_details "boom"})- (t2/update! :model/WorkspaceDatabase {:workspace_id id} {:status_details "db boom"})- (is (=? {:status :database-provisioning-failure, :status_details "boom"}- (t2/select-one :model/Workspace :id id)))- (is (= "db boom"- (t2/select-one-fn :status_details :model/WorkspaceDatabase :workspace_id id)))))))+ (let [{id :id} (create-ws!+ {:name "Evolving"+ :databases [(ws-db-attrs {:status :unprovisioned})]})+ updated (workspace/update-workspace!+ id+ {:name "Evolving"+ :databases [(ws-db-attrs {:status :provisioned})]})]+ (is (= :provisioned (:status (first (:databases updated)))))))))++(deftest update-preserves-initialized-rows-test+ (testing "update-workspace! leaves :provisioned rows untouched when the PUT keeps their database_id + input"+ (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS"}+ :model/WorkspaceDatabase {init-id :id}+ {:workspace_id ws-id+ :database_id (mt/id)+ :database_details {:user "keep-me" :password "keep-pw"}+ :output_namespace "keep_schema"+ :input_schemas ["public"]+ :status :provisioned}]+ (let [updated (workspace/update-workspace!+ ws-id+ {:name "Renamed"+ :databases [{:database_id (mt/id) :input_schemas ["public"]}]})+ row (t2/select-one :model/WorkspaceDatabase :id init-id)]+ (testing "name update takes effect"+ (is (= "Renamed" (:name updated))))+ (testing "the initialized row was not deleted + reinserted (its id is the same)"+ (is (some? row))+ (is (= :provisioned (:status row))))+ (testing "credentials and output_namespace are preserved verbatim"+ (is (= {:user "keep-me" :password "keep-pw"} (:database_details row)))+ (is (= "keep_schema" (:output_namespace row))))))))++(deftest update-rejects-dropping-initialized-test+ (testing "update-workspace! refuses to drop an :provisioned row"+ (mt/with-temp [:model/Database {db2-id :id} {:engine :h2 :details {}}+ :model/Workspace {ws-id :id} {:name "WS"}+ :model/WorkspaceDatabase _+ {:workspace_id ws-id+ :database_id (mt/id)+ :database_details {:user "u"}+ :output_namespace "sch"+ :input_schemas ["public"]+ :status :provisioned}]+ (is (thrown-with-msg?+ Exception+ #"provisioned"+ (workspace/update-workspace!+ ws-id+ {:name "WS" :databases [{:database_id db2-id :input_schemas ["public"]}]}))))))++(deftest update-rejects-changing-initialized-input-test+ (testing "update-workspace! refuses to change :input_schemasof an :provisioned row"+ (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS"}+ :model/WorkspaceDatabase _+ {:workspace_id ws-id+ :database_id (mt/id)+ :database_details {:user "u"}+ :output_namespace "sch"+ :input_schemas ["public"]+ :status :provisioned}]+ (is (thrown-with-msg?+ Exception+ #"provisioned"+ (workspace/update-workspace!+ ws-id+ {:name "WS"+ :databases [{:database_id (mt/id) :input_schemas ["analytics"]}]}))))))(deftest cascade-delete-database-test(testing "Deleting an underlying Database cascades to workspace_database rows"diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/provisioning_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/provisioning_test.cljdeleted file mode 100644index fddd54c61b8..00000000000--- a/enterprise/backend/test/metabase_enterprise/workspaces/provisioning_test.clj+++ /dev/null@@ -1,443 +0,0 @@-(ns ^:synchronous metabase-enterprise.workspaces.provisioning-test- "Tests for the workspace provisioning lifecycle: [[provisioning/provision-workspace!]]- and [[provisioning/deprovision-workspace!]] driving the workspace and- workspace_database statuses. `use-fixtures`-installed stubs replace the real- DatabaseProvisioner and InstanceProvisioner for every test (hence- `^:synchronous`); individual tests override them with their own reify via an- inner `with-redefs`."- (:require- [clojure.test :refer [deftest is testing use-fixtures]]- [metabase-enterprise.remote-sync.core :as remote-sync]- [metabase-enterprise.workspaces.provisioning :as provisioning]- [metabase-enterprise.workspaces.provisioning.database :as provisioning.database]- [metabase-enterprise.workspaces.provisioning.instance :as provisioning.instance]- [metabase.test :as mt]- [metabase.test.fixtures :as fixtures]- [toucan2.core :as t2])- (:import- (clojure.lang ExceptionInfo)))--(set! *warn-on-reflection* true)--(def ^:private database-calls- "init!/destroy! calls recorded by the default stub DatabaseProvisioner. Reset per test."- (atom []))--(defn- stub-database-provisioner- "DatabaseProvisioner that records init!/destroy! calls in [[database-calls]] and never- touches a warehouse."- []- (reify provisioning.database/DatabaseProvisioner- (details [_ _ _ workspace]- {:schema (str "mb_iso_" (:id workspace)) :database_details {:user "stub_user"}})- (init! [_ _ db _] (swap! database-calls conj [:init! (:id db)]) nil)- (grant! [_ _ _ _ _] nil)- (destroy! [_ _ db _] (swap! database-calls conj [:destroy! (:id db)]) nil)))--(defn- stub-instance-provisioner- "InstanceProvisioner that never talks to a real service and starts instantly."- []- (reify provisioning.instance/InstanceProvisioner- (create! [_ _workspace _config]- {:id (str (random-uuid)) :url "https://example.com" :status :active})- (fetch [_ instance-id]- {:id instance-id :url "https://example.com" :status :active})- (delete! [_ _instance-id] nil)))--(use-fixtures :once- (fixtures/initialize :db)- (fn [thunk]- (with-redefs [provisioning.database/database-provisioner (stub-database-provisioner)- provisioning.instance/instance-provisioner (stub-instance-provisioner)]- (thunk))))--(use-fixtures :each- (fn [thunk]- (reset! database-calls [])- (thunk)))--(defn- failing-on- "Like [[stub-database-provisioner]] but `op` (:init! or :destroy!) throws for database- `db-id`."- [op db-id]- (reify provisioning.database/DatabaseProvisioner- (details [_ _ _ workspace]- {:schema (str "mb_iso_" (:id workspace)) :database_details {:user "stub_user"}})- (init! [_ _ db _]- (when (and (= op :init!) (= (:id db) db-id))- (throw (ex-info "boom" {}))))- (grant! [_ _ _ _ _] nil)- (destroy! [_ _ db _]- (when (and (= op :destroy!) (= (:id db) db-id))- (throw (ex-info "warehouse down" {}))))))--(defn- wsd-attrs [ws-id db-id]- {:workspace_id ws-id- :database_id db-id- :database_details {}- :output_namespace ""- :input_schemas ["public"]})--(defn- workspace-row [ws-id]- (t2/select-one :model/Workspace :id ws-id))--(deftest provision-workspace-success-test- (testing "provision-workspace! provisions every database and lands the workspace :provisioned"- (mt/with-temp [:model/Database {db2-id :id} {:engine :h2 :details {}}- :model/Workspace {ws-id :id} {:name "WS"}- :model/WorkspaceDatabase {wsd1-id :id} (wsd-attrs ws-id (mt/id))- :model/WorkspaceDatabase {wsd2-id :id} (wsd-attrs ws-id db2-id)]- (is (=? {:status :provisioned :status_details nil}- (provisioning/provision-workspace! (workspace-row ws-id)))- "the updated workspace copy is returned")- (is (=? {:status :provisioned- :status_details nil- :instance_id string?- :instance_url "https://example.com"}- (workspace-row ws-id))- "the stub instance provisioner persisted the instance identifiers")- (is (=? {:status :provisioned- :status_details nil- :output_namespace (str "mb_iso_" wsd1-id)- :database_details {:user "stub_user"}}- (t2/select-one :model/WorkspaceDatabase :id wsd1-id)))- (is (=? {:status :provisioned- :output_namespace (str "mb_iso_" wsd2-id)}- (t2/select-one :model/WorkspaceDatabase :id wsd2-id))))))--(deftest provision-workspace-branch-test- (testing "the branch phase creates the workspace's target branch when remote sync is enabled"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS" :target_branch "ws-branch"}]- (let [calls (atom [])]- (with-redefs [remote-sync/remote-sync-enabled (constantly true)- remote-sync/branch-exists? (fn [branch] (swap! calls conj [:exists? branch]) false)- remote-sync/create-branch! (fn [branch] (swap! calls conj [:create! branch]) branch)]- (is (=? {:status :provisioned}- (provisioning/provision-workspace! (workspace-row ws-id))))- (is (= [[:exists? "ws-branch"] [:create! "ws-branch"]] @calls))))))- (testing "an already existing branch is fine"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS" :target_branch "ws-branch"}]- (with-redefs [remote-sync/remote-sync-enabled (constantly true)- remote-sync/branch-exists? (constantly true)- remote-sync/create-branch! (fn [_] (throw (ex-info "should not be called" {})))]- (is (=? {:status :provisioned}- (provisioning/provision-workspace! (workspace-row ws-id)))))))- (testing "the branch phase is a no-op when remote sync is disabled or there is no branch"- (mt/with-temp [:model/Workspace {with-branch-id :id} {:name "WS" :target_branch "ws-branch"}- :model/Workspace {no-branch-id :id} {:name "WS"}]- (with-redefs [remote-sync/remote-sync-enabled (constantly false)- remote-sync/branch-exists? (fn [_] (throw (ex-info "should not be called" {})))- remote-sync/create-branch! (fn [_] (throw (ex-info "should not be called" {})))]- (is (=? {:status :provisioned}- (provisioning/provision-workspace! (workspace-row with-branch-id))))- (is (=? {:status :provisioned}- (provisioning/provision-workspace! (workspace-row no-branch-id)))))))- (testing "a branch failure lands in :branch-provisioning-failure and stops the run"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS" :target_branch "ws-branch"}]- (with-redefs [remote-sync/remote-sync-enabled (constantly true)- remote-sync/branch-exists? (constantly false)- remote-sync/create-branch! (fn [_] (throw (ex-info "git down" {})))]- (is (thrown-with-msg? ExceptionInfo #"git down"- (provisioning/provision-workspace! (workspace-row ws-id))))- (is (=? {:status :branch-provisioning-failure- :status_details "git down"}- (workspace-row ws-id)))))))--(deftest deprovision-workspace-branch-test- (testing "the branch phase deletes the branch and keeps :target_branch as is"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS"- :status :provisioned- :target_branch "ws-branch"}]- (let [calls (atom [])]- (with-redefs [remote-sync/remote-sync-enabled (constantly true)- remote-sync/delete-branch! (fn [branch] (swap! calls conj [:delete! branch]) nil)]- (is (=? {:status :unprovisioned}- (provisioning/deprovision-workspace! (workspace-row ws-id))))- (is (= [[:delete! "ws-branch"]] @calls))- (is (= "ws-branch" (:target_branch (workspace-row ws-id)))- "the branch column survives deprovisioning")))))- (testing "when remote sync is disabled there is nothing to delete and the column is kept"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS"- :status :provisioned- :target_branch "ws-branch"}]- (with-redefs [remote-sync/remote-sync-enabled (constantly false)- remote-sync/delete-branch! (fn [_] (throw (ex-info "should not be called" {})))]- (is (=? {:status :unprovisioned}- (provisioning/deprovision-workspace! (workspace-row ws-id))))- (is (= "ws-branch" (:target_branch (workspace-row ws-id)))))))- (testing "a branch deletion failure lands in :branch-deprovisioning-failure and stops the run"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS"- :status :provisioned- :target_branch "ws-branch"}]- (with-redefs [remote-sync/remote-sync-enabled (constantly true)- remote-sync/delete-branch! (fn [_] (throw (ex-info "git down" {})))]- (is (thrown-with-msg? ExceptionInfo #"git down"- (provisioning/deprovision-workspace! (workspace-row ws-id))))- (is (=? {:status :branch-deprovisioning-failure- :status_details "git down"}- (workspace-row ws-id)))))))--(deftest provision-workspace-failure-test- (testing "the first database failure stops the run and records the error on the row and the workspace"- (mt/with-temp [:model/Database {db2-id :id} {:engine :h2 :details {}}- :model/Database {db3-id :id} {:engine :h2 :details {}}- :model/Workspace {ws-id :id} {:name "WS"}- :model/WorkspaceDatabase {wsd1-id :id} (wsd-attrs ws-id (mt/id))- :model/WorkspaceDatabase {wsd2-id :id} (wsd-attrs ws-id db2-id)- :model/WorkspaceDatabase {wsd3-id :id} (wsd-attrs ws-id db3-id)]- (with-redefs [provisioning.database/database-provisioner (failing-on :init! db2-id)]- (is (thrown-with-msg? ExceptionInfo #"boom"- (provisioning/provision-workspace! (workspace-row ws-id)))- "the failure is rethrown"))- (is (=? {:status :database-provisioning-failure- :status_details "boom"- :instance_id nil}- (workspace-row ws-id))- "the instance phase is never reached after a database failure")- (is (=? {:status :provisioned}- (t2/select-one :model/WorkspaceDatabase :id wsd1-id))- "the database provisioned before the failure stays :provisioned")- (is (=? {:status :provisioning-failure :status_details "boom"}- (t2/select-one :model/WorkspaceDatabase :id wsd2-id))- "the failed database records the failure — no rollback")- (is (=? {:status :unprovisioned}- (t2/select-one :model/WorkspaceDatabase :id wsd3-id))- "databases after the failure are not attempted"))))--(deftest provision-workspace-retry-test- (testing "a retry after a failure skips already-:provisioned databases and finishes the job"- (mt/with-temp [:model/Database {db2-id :id} {:engine :h2 :details {}}- :model/Workspace {ws-id :id} {:name "WS"}- :model/WorkspaceDatabase {wsd1-id :id} (wsd-attrs ws-id (mt/id))- :model/WorkspaceDatabase {wsd2-id :id} (wsd-attrs ws-id db2-id)]- (with-redefs [provisioning.database/database-provisioner (failing-on :init! db2-id)]- (is (thrown-with-msg? ExceptionInfo #"boom"- (provisioning/provision-workspace! (workspace-row ws-id)))))- (reset! database-calls [])- (provisioning/provision-workspace! (workspace-row ws-id))- (is (= [[:init! db2-id]] @database-calls)- "only the previously-failed database is re-provisioned")- (is (=? {:status :provisioned :status_details nil} (workspace-row ws-id)))- (is (=? {:status :provisioned :status_details nil}- (t2/select-one :model/WorkspaceDatabase :id wsd2-id))- "the retried row ends :provisioned with its failure details cleared")- (is (=? {:status :provisioned}- (t2/select-one :model/WorkspaceDatabase :id wsd1-id))))))--(deftest provision-workspace-instance-failure-and-retry-test- (testing "an instance-provisioning failure lands in :instance-provisioning-failure and a retry finishes"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS"}- :model/WorkspaceDatabase {wsd-id :id} (wsd-attrs ws-id (mt/id))]- (with-redefs [provisioning.instance/instance-provisioner- (reify provisioning.instance/InstanceProvisioner- (create! [_ _ _] (throw (ex-info "no capacity" {})))- (fetch [_ id] {:id id :url "https://example.com" :status :active})- (delete! [_ _] nil))]- (is (thrown-with-msg? ExceptionInfo #"no capacity"- (provisioning/provision-workspace! (workspace-row ws-id)))))- (is (=? {:status :instance-provisioning-failure- :status_details "no capacity"- :instance_id nil}- (workspace-row ws-id)))- (is (=? {:status :provisioned}- (t2/select-one :model/WorkspaceDatabase :id wsd-id))- "the databases were provisioned before the instance phase failed")- (testing "retry skips the already-provisioned databases and finishes the instance"- (reset! database-calls [])- (provisioning/provision-workspace! (workspace-row ws-id))- (is (= [] @database-calls))- (is (=? {:status :provisioned :status_details nil :instance_id string?}- (workspace-row ws-id)))))))--(deftest deprovision-workspace-instance-failure-test- (testing "an instance-deprovisioning failure lands in :instance-deprovisioning-failure and stops"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS"}- :model/WorkspaceDatabase {wsd-id :id} (wsd-attrs ws-id (mt/id))]- (provisioning/provision-workspace! (workspace-row ws-id))- (with-redefs [provisioning.instance/instance-provisioner- (reify provisioning.instance/InstanceProvisioner- (create! [_ _ _] {:id "x" :url "https://example.com" :status :active})- (fetch [_ id] {:id id :url "https://example.com" :status :active})- (delete! [_ _] (throw (ex-info "instance stuck" {}))))]- (is (thrown-with-msg? ExceptionInfo #"instance stuck"- (provisioning/deprovision-workspace! (workspace-row ws-id)))))- (is (=? {:status :instance-deprovisioning-failure- :status_details "instance stuck"- :instance_id string?}- (workspace-row ws-id))- "the instance identifiers are kept for the retry")- (is (=? {:status :provisioned}- (t2/select-one :model/WorkspaceDatabase :id wsd-id))- "the database phase is never reached after an instance failure")- (testing "retry once the instance is deletable"- (provisioning/deprovision-workspace! (workspace-row ws-id))- (is (=? {:status :unprovisioned :instance_id nil}- (workspace-row ws-id)))))))--(deftest provision-instance-receives-workspace-config-test- (testing "the instance provisioner receives the workspace's config-file map"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS"- :creator_id (mt/user->id :crowberto)}- :model/WorkspaceDatabase _ (assoc (wsd-attrs ws-id (mt/id))- :status :provisioned- :output_namespace "mb_iso_x")]- (let [received (atom nil)]- (with-redefs [provisioning.instance/instance-provisioner- (reify provisioning.instance/InstanceProvisioner- (create! [_ _ config] (reset! received config) {:id "i" :url "https://example.com" :status :active})- (fetch [_ id] {:id id :url "https://example.com" :status :active})- (delete! [_ _] nil))]- (provisioning/provision-workspace! (workspace-row ws-id)))- (is (=? {:version 1- :config {:workspace {:name "WS"}}}- @received)- "create! received the workspace's config map")))))--(deftest set-initial-workspace-status-guard-test- (testing "set-initial-workspace-*-status! atomically refuse to start a run while one is in flight"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "Busy" :status :database-provisioning}]- (doseq [start! [provisioning/set-initial-workspace-provisioning-status!- provisioning/set-initial-workspace-deprovisioning-status!]]- (is (= 400 (try (start! (workspace-row ws-id))- (catch Exception e (:status-code (ex-data e))))))- (is (= :database-provisioning (:status (workspace-row ws-id)))- "the in-flight status is untouched"))))- (testing "and start the run from any other settled status"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "Settled" :status :database-provisioning-failure}]- (is (=? {:status :database-provisioning}- (provisioning/set-initial-workspace-provisioning-status! (workspace-row ws-id))))- (is (= :database-provisioning (:status (workspace-row ws-id))))))- (testing "a provision cannot start when the workspace is already :provisioned"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "Done" :status :provisioned}]- (is (= 400 (try (provisioning/set-initial-workspace-provisioning-status! (workspace-row ws-id))- (catch Exception e (:status-code (ex-data e))))))- (is (= :provisioned (:status (workspace-row ws-id))))))- (testing "a deprovision cannot start when the workspace is already :unprovisioned"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "Empty" :status :unprovisioned}]- (is (= 400 (try (provisioning/set-initial-workspace-deprovisioning-status! (workspace-row ws-id))- (catch Exception e (:status-code (ex-data e))))))- (is (= :unprovisioned (:status (workspace-row ws-id)))))))--(deftest provision-instance-polls-until-running-test- (testing "provision-instance! records the ids up front, then polls until the instance is :active"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS"}]- (let [statuses (atom [:creating :creating :active])- polls (atom 0)]- (with-redefs [provisioning.instance/instance-poll-interval-ms 1- provisioning.instance/instance-provisioner- (reify provisioning.instance/InstanceProvisioner- (create! [_ _ _] {:id "async-1" :url "https://example.com" :status :creating})- (fetch [_ id]- (swap! polls inc)- {:id id :url "https://example.com" :status (ffirst (swap-vals! statuses rest))})- (delete! [_ _] nil))]- (provisioning/provision-workspace! (workspace-row ws-id)))- (is (= 3 @polls) "polling continues through :creating until the terminal status")- (is (=? {:status :provisioned :instance_id "async-1" :instance_url "https://example.com"}- (workspace-row ws-id)))))))--(deftest provision-instance-error-status-test- (testing "an instance that lands in :error fails the run but keeps the recorded ids"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS"}]- (with-redefs [provisioning.instance/instance-provisioner- (reify provisioning.instance/InstanceProvisioner- (create! [_ _ _] {:id "doomed-1" :url "https://example.com" :status :creating})- (fetch [_ id] {:id id :url "https://example.com" :status :error})- (delete! [_ _] nil))]- (is (thrown-with-msg? ExceptionInfo #"failed to start"- (provisioning/provision-workspace! (workspace-row ws-id)))))- (is (=? {:status :instance-provisioning-failure- :status_details "Workspace instance failed to start"- :instance_id "doomed-1"}- (workspace-row ws-id))- "the instance identifiers stay recorded for the retry's cleanup"))))--(deftest provision-workspace-noop-when-provisioned-test- (testing "provisioning a fully-provisioned workspace is a no-op"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS"}- :model/WorkspaceDatabase _ (assoc (wsd-attrs ws-id (mt/id))- :status :provisioned- :output_namespace "mb_iso_x")]- (provisioning/provision-workspace! (workspace-row ws-id))- (is (= [] @database-calls) "no provisioner database-calls are made")- (is (=? {:status :provisioned} (workspace-row ws-id))))))--(deftest deprovision-workspace-success-test- (testing "deprovision-workspace! tears down every database and lands the workspace :unprovisioned"- (mt/with-temp [:model/Database {db2-id :id} {:engine :h2 :details {}}- :model/Workspace {ws-id :id} {:name "WS"}- :model/WorkspaceDatabase {wsd1-id :id} (wsd-attrs ws-id (mt/id))- :model/WorkspaceDatabase {wsd2-id :id} (wsd-attrs ws-id db2-id)]- (provisioning/provision-workspace! (workspace-row ws-id))- (reset! database-calls [])- (is (=? {:status :unprovisioned :status_details nil :instance_id nil}- (provisioning/deprovision-workspace! (workspace-row ws-id)))- "the updated workspace copy is returned")- (is (= [[:destroy! (mt/id)] [:destroy! db2-id]] @database-calls)- "every database gets a warehouse teardown, in row order")- (is (=? {:status :unprovisioned- :status_details nil- :instance_id nil- :instance_url nil}- (workspace-row ws-id))- "the instance identifiers are cleared")- (doseq [wsd-id [wsd1-id wsd2-id]]- (is (=? {:status :unprovisioned- :status_details nil- :output_namespace ""- :database_details {}}- (t2/select-one :model/WorkspaceDatabase :id wsd-id))- "the persisted warehouse identifiers are cleared")))))--(deftest deprovision-workspace-failure-and-retry-test- (testing "a teardown failure records the error and a later retry finishes the job"- (mt/with-temp [:model/Database {db2-id :id} {:engine :h2 :details {}}- :model/Workspace {ws-id :id} {:name "WS"}- :model/WorkspaceDatabase {wsd1-id :id} (wsd-attrs ws-id (mt/id))- :model/WorkspaceDatabase {wsd2-id :id} (wsd-attrs ws-id db2-id)]- (provisioning/provision-workspace! (workspace-row ws-id))- (with-redefs [provisioning.database/database-provisioner (failing-on :destroy! (mt/id))]- (is (thrown-with-msg? ExceptionInfo #"warehouse down"- (provisioning/deprovision-workspace! (workspace-row ws-id)))))- (is (=? {:status :database-deprovisioning-failure :status_details "warehouse down"}- (workspace-row ws-id)))- (is (=? {:status :deprovisioning-failure :status_details "warehouse down"}- (t2/select-one :model/WorkspaceDatabase :id wsd1-id)))- (is (=? {:status :provisioned}- (t2/select-one :model/WorkspaceDatabase :id wsd2-id))- "databases after the failure are not attempted")- (testing "retry once the warehouse is reachable"- (provisioning/deprovision-workspace! (workspace-row ws-id))- (is (=? {:status :unprovisioned :status_details nil} (workspace-row ws-id)))- (doseq [wsd-id [wsd1-id wsd2-id]]- (is (=? {:status :unprovisioned :status_details nil}- (t2/select-one :model/WorkspaceDatabase :id wsd-id))))))))--(deftest deprovision-workspace-noop-when-unprovisioned-test- (testing "deprovisioning an :unprovisioned workspace is a no-op"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS"}- :model/WorkspaceDatabase _ (wsd-attrs ws-id (mt/id))]- (provisioning/deprovision-workspace! (workspace-row ws-id))- (is (= [] @database-calls) "no provisioner database-calls are made")- (is (=? {:status :unprovisioned} (workspace-row ws-id))))))--(deftest deprovision-recomputes-identifiers-for-crashed-provisioning-row-test- (testing "a crashed :provisioning row that never persisted its identifiers is torn down- with identifiers recomputed via the provisioner's `details`"- (mt/with-temp [:model/Workspace {ws-id :id} {:name "WS"}- :model/WorkspaceDatabase {wsd-id :id} (assoc (wsd-attrs ws-id (mt/id))- :status :provisioning)]- (let [destroyed (atom nil)- recording (reify provisioning.database/DatabaseProvisioner- (details [_ _ _ workspace]- {:schema (str "mb_iso_" (:id workspace)) :database_details {:user "recomputed"}})- (init! [_ _ _ _] nil)- (grant! [_ _ _ _ _] nil)- (destroy! [_ _ _ workspace] (reset! destroyed workspace) nil))]- (with-redefs [provisioning.database/database-provisioner recording]- (provisioning/deprovision-workspace! (workspace-row ws-id)))- (is (=? {:status :unprovisioned} (workspace-row ws-id)))- (is (=? {:schema (str "mb_iso_" wsd-id)- :database_details {:user "recomputed"}}- @destroyed)- "destroy! received the deterministically recomputed identifiers")))))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/query_processor/corpus_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/query_processor/corpus_test.cljindex 016e759a9af..6a6a132d8eb 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/query_processor/corpus_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/query_processor/corpus_test.clj@@ -15,8 +15,7 @@[clojure.string :as str][clojure.test :refer :all][metabase-enterprise.advanced-config.file.workspace :as advanced-config.file.workspace]- [metabase-enterprise.workspaces.instance :as ws.instance]- [metabase-enterprise.workspaces.provisioning.database :as provisioning.database]+ [metabase-enterprise.workspaces.core :as ws][metabase-enterprise.workspaces.table-remapping :as ws.table-remapping][metabase-enterprise.workspaces.test-util :as workspaces.tu][metabase.driver :as driver]@@ -301,7 +300,7 @@;; The from-spec handed to the rewriter must match what the driver;; actually *emits* in SQL, which is governed by qualified-name-components.emitted-slots (set (driver/qualified-name-components driver))- input-positions (provisioning.database/engine-namespace-positions fake-db {:schema first-input})+ input-positions (ws/engine-namespace-positions fake-db {:schema first-input})from-spec {:db (if (:db emitted-slots) (or (:db input-positions) "") ""):schema (if (:schema emitted-slots) (or (:schema input-positions) "") ""):table source-table}]@@ -312,7 +311,7 @@;; 1. Load through the production loader. Populates the atom.(advanced-config.file.workspace/apply-workspace-section! section);; 2. Read the namespace via the production reader.- (let [ws-ns (ws.instance/db-workspace-namespace db-id)+ (let [ws-ns (ws/db-workspace-namespace db-id)to-spec (merge from-spec ws-ns){:keys [tables rewritten]} (rewrite-and-parsedriver canonical-sql@@ -345,4 +344,4 @@(is (re-find (re-pattern (java.util.regex.Pattern/quote output-db)) from-text)(str "expected " (pr-str output-db) " in FROM; got: " from-text)))))))(finally- (ws.instance/clear-instance-workspace!)))))))))))+ (ws/clear-instance-workspace!)))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/provisioning/database_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/remapping_cleanup_test.cljsimilarity index 53%rename from enterprise/backend/test/metabase_enterprise/workspaces/provisioning/database_test.cljrename to enterprise/backend/test/metabase_enterprise/workspaces/remapping_cleanup_test.cljindex 3bd12cce032..7bde4dd283b 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/provisioning/database_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/remapping_cleanup_test.clj@@ -1,14 +1,15 @@-(ns metabase-enterprise.workspaces.provisioning.database-test- "Tests for the iso-scoped `TableRemapping` deletion path- ([[provisioning.database/clear-mappings-for-iso!]]) invoked from- [[provisioning.database/deprovision-database!]]."+(ns metabase-enterprise.workspaces.remapping-cleanup-test+ "Tests for [[metabase-enterprise.workspaces.remapping-cleanup]] — the iso-scoped+ `TableRemapping` deletion path invoked from `provisioning/deprovision-workspace-database!`."(:require[clojure.test :refer :all]- [metabase-enterprise.workspaces.provisioning.database :as provisioning.database]+ [metabase-enterprise.workspaces.provisioning :as provisioning]+ [metabase-enterprise.workspaces.remapping-cleanup :as ws.remapping-cleanup][metabase-enterprise.workspaces.table-remapping :as ws.table-remapping][metabase.driver :as driver][metabase.driver.sql :as driver.sql][metabase.test :as mt]+ [metabase.test.util.thread-local :as tu.thread-local][toucan2.core :as t2]));;; Register a fake `db-schema-table`-shape engine so this test can exercise the@@ -48,7 +49,7 @@;; Deprovisioning workspace A: pass a fake Database row that won't trigger the;; 3-slot :db branch — Postgres-shaped, no :details :db. iso :db slot stays "".- (let [n (provisioning.database/clear-mappings-for-iso!+ (let [n (ws.remapping-cleanup/clear-mappings-for-iso!{:engine :postgres :details {}}(mt/id)"ws_a")]@@ -66,13 +67,13 @@(mt/id)(fn [](is (= 0- (provisioning.database/clear-mappings-for-iso!+ (ws.remapping-cleanup/clear-mappings-for-iso!{:engine :postgres :details {}}(mt/id)"never_provisioned"))"no rows, returns 0")(is (= 0- (provisioning.database/clear-mappings-for-iso!+ (ws.remapping-cleanup/clear-mappings-for-iso!{:engine :postgres :details {}}(mt/id)""))@@ -93,7 +94,7 @@{:db "AnalyticsDB" :schema "dbo" :table "products"}{:db "AnalyticsDB" :schema "ws_b" :table "products_copy"})- (let [n (provisioning.database/clear-mappings-for-iso!+ (let [n (ws.remapping-cleanup/clear-mappings-for-iso!{:engine ::fake-3-slot :details {:db "AnalyticsDB"}}(mt/id)"ws_a")]@@ -106,41 +107,46 @@(deftest deprovision-clears-rows-even-when-destroy-throws-test(testing "When the driver's `destroy!` step throws (e.g. BQ dataset gone but SA delete fails),- `deprovision-database!` must still clear the workspace's `TableRemapping`+ `deprovision-workspace-database!` must still clear the workspace's `TableRemapping`rows. Without that, future queries against canonical tables on this DB rewrite toan iso namespace that no longer exists on the warehouse and 500 in the QP. Thewarehouse-side leak (orphan SA / dataset) is acceptable -- app-DB state must matchthe deprovision intent so the workspace stops routing queries."- (mt/with-temp [:model/Database {db-id :id} {:engine :postgres :details {}}- :model/Workspace {ws-id :id} {:name "tear-down"- :creator_id (mt/user->id :crowberto)}- :model/WorkspaceDatabase {wsd-id :id :as wsd} {:workspace_id ws-id- :database_id db-id- :input_schemas ["public"]- :database_details {}- :output_namespace "ws_alice"- :status :deprovisioning}]- (ws.table-remapping/add-mapping!- db-id {:schema "public" :table "orders"} {:schema "ws_alice" :table "orders_copy"})- (ws.table-remapping/add-mapping!- db-id {:schema "public" :table "products"} {:schema "ws_alice" :table "products_copy"})- (is (= 2 (count (ws.table-remapping/all-mappings-for-db db-id)))- "fixture: two remap rows registered before deprovision")- ;; Provisioner that fails on destroy! to simulate partial warehouse teardown- ;; (e.g. BQ dataset deleted, SA delete throws).- (let [failing-provisioner (reify provisioning.database/DatabaseProvisioner- (details [_ _ _ _] {:schema "ws_alice" :database_details {}})- (init! [_ _ _ _] (throw (ex-info "not used" {})))- (grant! [_ _ _ _ _] (throw (ex-info "not used" {})))- (destroy! [_ _ _ _] (throw (ex-info "warehouse teardown blew up" {}))))]- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"warehouse teardown blew up"- (provisioning.database/deprovision-database! wsd failing-provisioner))- "deprovision rethrows the destroy failure so the caller knows"))- (is (zero? (count (ws.table-remapping/all-mappings-for-db db-id)))- "remap rows must be cleared even when destroy! threw -- otherwise canonical-table queries 500")- (is (=? {:status :deprovisioning-failure- :status_details "warehouse teardown blew up"}- (t2/select-one :model/WorkspaceDatabase :id wsd-id))- "the failed deprovision is recorded on the row for retry"))))+ ;; Disable the rollback-only tx wrap that `metabase.test.redefs` adds around `with-temp`.+ ;; `deprovision-workspace-database!` rebinds `*current-connectable*` to nil for the+ ;; `TableRemapping` cleanup so the DELETE survives a `with-cluster-lock` rollback when+ ;; `destroy!` throws. Under the rollback-only wrap, that fresh connection can't see+ ;; the fixture rows -- they live only inside the wrap's open transaction. Real+ ;; production callers persist `TableRemapping` rows before deprovision runs.+ (binding [tu.thread-local/*thread-local* false]+ (mt/with-temp [:model/Database {db-id :id} {:engine :postgres :details {}}+ :model/Workspace {ws-id :id} {:name "tear-down"+ :creator_id (mt/user->id :crowberto)}+ :model/WorkspaceDatabase {wsd-id :id} {:workspace_id ws-id+ :database_id db-id+ :input_schemas ["public"]+ :database_details {}+ :output_namespace "ws_alice"+ :status :deprovisioning}]+ (ws.table-remapping/add-mapping!+ db-id {:schema "public" :table "orders"} {:schema "ws_alice" :table "orders_copy"})+ (ws.table-remapping/add-mapping!+ db-id {:schema "public" :table "products"} {:schema "ws_alice" :table "products_copy"})+ (is (= 2 (count (ws.table-remapping/all-mappings-for-db db-id)))+ "fixture: two remap rows registered before deprovision")+ ;; Provisioner that fails on destroy! to simulate partial warehouse teardown+ ;; (e.g. BQ dataset deleted, SA delete throws).+ (let [failing-provisioner (reify provisioning/Provisioner+ (details [_ _ _ _] (throw (ex-info "not used" {})))+ (init! [_ _ _ _] (throw (ex-info "not used" {})))+ (grant! [_ _ _ _ _] (throw (ex-info "not used" {})))+ (destroy! [_ _ _ _] (throw (ex-info "warehouse teardown blew up" {}))))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"warehouse teardown blew up"+ (provisioning/deprovision-workspace-database! wsd-id failing-provisioner))+ "deprovision rethrows the destroy failure so the caller knows"))+ (is (zero? (count (ws.table-remapping/all-mappings-for-db db-id)))+ "remap rows must be cleared even when destroy! threw -- otherwise canonical-table queries 500")+ (is (= :provisioned (:status (t2/select-one :model/WorkspaceDatabase :id wsd-id)))+ "current rollback semantic: failed deprovision flips status back to :provisioned")))))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/settings_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/settings_test.cljindex d375b6daef9..b126ab64c5f 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/settings_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/settings_test.clj@@ -1,7 +1,7 @@(ns metabase-enterprise.workspaces.settings-test(:require[clojure.test :refer [deftest is testing use-fixtures]]- [metabase-enterprise.workspaces.instance :as ws.instance]+ [metabase-enterprise.workspaces.core :as ws][metabase.test.fixtures :as fixtures]))(use-fixtures :once (fixtures/initialize :db))@@ -10,23 +10,23 @@(try(thunk)(finally- (ws.instance/clear-instance-workspace!)))))+ (ws/clear-instance-workspace!)))))(deftest workspace-mode?-false-when-setting-empty-test(testing "workspace-mode? is false when no :workspace section has been loaded"- (ws.instance/clear-instance-workspace!)- (is (false? (ws.instance/workspace-mode?)))))+ (ws/clear-instance-workspace!)+ (is (false? (ws/workspace-mode?)))))(deftest workspace-mode?-true-when-setting-populated-test(testing "workspace-mode? flips to true once a workspace config has been set"- (ws.instance/clear-instance-workspace!)- (is (false? (ws.instance/workspace-mode?)))- (ws.instance/set-instance-workspace! {:name "Acme" :databases {}})- (is (true? (ws.instance/workspace-mode?)))))+ (ws/clear-instance-workspace!)+ (is (false? (ws/workspace-mode?)))+ (ws/set-instance-workspace! {:name "Acme" :databases {}})+ (is (true? (ws/workspace-mode?)))))(deftest workspace-mode?-flips-back-to-false-on-clear-test(testing "workspace-mode? returns to false after clear-instance-workspace!"- (ws.instance/set-instance-workspace! {:name "Acme" :databases {}})- (is (true? (ws.instance/workspace-mode?)))- (ws.instance/clear-instance-workspace!)- (is (false? (ws.instance/workspace-mode?)))))+ (ws/set-instance-workspace! {:name "Acme" :databases {}})+ (is (true? (ws/workspace-mode?)))+ (ws/clear-instance-workspace!)+ (is (false? (ws/workspace-mode?)))))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/table_remapping_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/table_remapping_test.cljindex 607c726991d..3435606d510 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/table_remapping_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/table_remapping_test.clj@@ -5,8 +5,7 @@(:require[clojure.test :refer :all][java-time.api :as t]- [metabase-enterprise.workspaces.instance :as ws.instance]- [metabase-enterprise.workspaces.provisioning.database :as provisioning.database]+ [metabase-enterprise.workspaces.core :as ws][metabase-enterprise.workspaces.table-remapping :as ws.table-remapping][metabase-enterprise.workspaces.test-util :as workspaces.tu][metabase.driver :as driver]@@ -35,12 +34,12 @@[db-id output-schema f](mt/with-premium-features #{:workspaces}(try- (ws.instance/set-instance-workspace! {:name "table-remapping-test-ws"- :databases {db-id {:input_schemas ["_"]- :output {:schema output-schema}}}})+ (ws/set-instance-workspace! {:name "table-remapping-test-ws"+ :databases {db-id {:input_schemas ["_"]+ :output {:schema output-schema}}}})(f)(finally- (ws.instance/clear-instance-workspace!)))))+ (ws/clear-instance-workspace!)))))(deftest remap-table-returns-nil-when-no-mapping-test(clean-db-fixture!@@ -144,12 +143,12 @@[db-id output-namespace f](mt/with-premium-features #{:workspaces}(try- (ws.instance/set-instance-workspace! {:name "ws-3-slot"- :databases {db-id {:input_schemas ["_"]- :output output-namespace}}})+ (ws/set-instance-workspace! {:name "ws-3-slot"+ :databases {db-id {:input_schemas ["_"]+ :output output-namespace}}})(f)(finally- (ws.instance/clear-instance-workspace!)))))+ (ws/clear-instance-workspace!)))))(deftest add-transform-target-mapping!-flows-both-slots-from-namespace-test(testing "When the workspace output namespace populates :db, both slots flow into the TableRemapping row"@@ -995,8 +994,8 @@(deftest engine-namespace-positions-unknown-driver-defaults-to-schema-table-test(testing "Unknown engine gets the default :schema-table shape -- no throw, no surprise"(is (= {:db nil :schema "public"}- (provisioning.database/engine-namespace-positions {:engine ::ws-test-unknown-driver :name "x"}- {:schema "public" :name "orders"})))))+ (ws/engine-namespace-positions {:engine ::ws-test-unknown-driver :name "x"}+ {:schema "public" :name "orders"})))));;; -------------------------- GHY-3553: MySQL-shape sentinel-leak regressions --------------------------;;;diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj b/enterprise/backend/test/metabase_enterprise/workspaces/test_util.cljindex e6a5fcaad5d..7245f8d03f6 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.workspaces.test-util"Shared test helpers for the workspaces module."(:require+ [metabase-enterprise.workspaces.core :as ws][metabase.driver :as driver]))(defn driver-loadable?@@ -13,3 +14,14 @@[engine](try (driver/the-initialized-driver engine) true(catch Throwable _ false)))++(defn with-workspace-locked-by-config+ "Flip the workspace-instance lock for the duration of `thunk`, restoring the+ prior value on exit. Reaches into the private atom via `#'` so the ws/core+ surface stays clean of test-only setters."+ [thunk]+ (let [a @#'ws/locked-by-config?*+ prior @a]+ (reset! a true)+ (try (thunk)+ (finally (reset! a prior)))))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/transform_hooks_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/transform_hooks_test.cljindex 0a4fbe06fc1..52b422019b6 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/transform_hooks_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/transform_hooks_test.clj@@ -1,7 +1,7 @@(ns metabase-enterprise.workspaces.transform-hooks-test(:require[clojure.test :refer [deftest is testing use-fixtures]]- [metabase-enterprise.workspaces.instance :as ws.instance]+ [metabase-enterprise.workspaces.core :as ws][metabase-enterprise.workspaces.table-remapping :as ws.table-remapping][metabase-enterprise.workspaces.transform-hooks :as ws.transform-hooks][metabase.test :as mt]@@ -13,7 +13,7 @@(deftest resolve-transform-target-no-workspace-passthrough-test(testing "without a provisioned WorkspaceDatabase, target passes through unchanged"- (with-redefs [ws.instance/db-workspace-namespace (constantly nil)+ (with-redefs [ws/db-workspace-namespace (constantly nil)ws.table-remapping/add-transform-target-mapping!(fn [& _] (throw (ex-info "add-transform-target-mapping! must not be called when no workspace is active" {})))](let [target {:schema "public" :name "orders" :type :table}]@@ -22,7 +22,7 @@(deftest resolve-transform-target-rewrites-schema-test(testing "with a provisioned WorkspaceDatabase, target's :schema and :name come from the recorded to-spec"(let [recorded (atom nil)]- (with-redefs [ws.instance/db-workspace-namespace (constantly {:schema "ws_alice"})+ (with-redefs [ws/db-workspace-namespace (constantly {:schema "ws_alice"})ws.table-remapping/add-transform-target-mapping!(fn [db-id target](reset! recorded {:db-id db-id :target target})@@ -40,7 +40,7 @@(deftest resolve-transform-target-records-remapping-test(testing "with a workspace active, the canonical->workspace remapping is recorded"(let [recorded (atom nil)]- (with-redefs [ws.instance/db-workspace-namespace (constantly {:schema "ws_alice"})+ (with-redefs [ws/db-workspace-namespace (constantly {:schema "ws_alice"})ws.table-remapping/add-transform-target-mapping!(fn [db-id target](reset! recorded {:db-id db-id :target target})@@ -57,12 +57,12 @@the way out."[output-schema body-fn](try- (ws.instance/set-instance-workspace! {:name "test-ws"- :databases {(mt/id) {:input_schemas ["_"]- :output {:schema output-schema}}}})+ (ws/set-instance-workspace! {:name "test-ws"+ :databases {(mt/id) {:input_schemas ["_"]+ :output {:schema output-schema}}}})(body-fn)(finally- (ws.instance/clear-instance-workspace!))))+ (ws/clear-instance-workspace!))))(deftest resolve-transform-target-end-to-end-test(testing "end-to-end against the test app DB: a remapping row appears after the hook fires"diff --git a/enterprise/frontend/src/metabase-enterprise/api/advanced-config.ts b/enterprise/frontend/src/metabase-enterprise/api/advanced-config.tsnew file mode 100644index 00000000000..fcf2e66bc7e--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/api/advanced-config.ts@@ -0,0 +1,28 @@+import type { ApplyAdvancedConfigRequest } from "metabase-types/api";++import { EnterpriseApi } from "./api";+import { invalidateTags, listTag, tag } from "./tags";++export const advancedConfigApi = EnterpriseApi.injectEndpoints({+ endpoints: (builder) => ({+ applyAdvancedConfig: builder.mutation<void, ApplyAdvancedConfigRequest>({+ query: ({ config }) => {+ const formData = new FormData();+ formData.append("config", config);+ return {+ method: "POST",+ url: "/api/ee/advanced-config",+ body: formData,+ };+ },+ invalidatesTags: (_, error) =>+ invalidateTags(error, [+ tag("workspace"),+ listTag("workspace"),+ listTag("database"),+ ]),+ }),+ }),+});++export const { useApplyAdvancedConfigMutation } = advancedConfigApi;diff --git a/enterprise/frontend/src/metabase-enterprise/api/index.ts b/enterprise/frontend/src/metabase-enterprise/api/index.tsindex b8b8e61be4f..141a5c307fb 100644--- a/enterprise/frontend/src/metabase-enterprise/api/index.ts+++ b/enterprise/frontend/src/metabase-enterprise/api/index.ts@@ -1,3 +1,4 @@+export * from "./advanced-config";export * from "./advanced-permissions";export * from "./ai-controls";export * from "./application-permissions";diff --git a/enterprise/frontend/src/metabase-enterprise/api/workspace-instance.ts b/enterprise/frontend/src/metabase-enterprise/api/workspace-instance.tsindex c19bb29256d..be0ecd73a20 100644--- a/enterprise/frontend/src/metabase-enterprise/api/workspace-instance.ts+++ b/enterprise/frontend/src/metabase-enterprise/api/workspace-instance.ts@@ -6,7 +6,7 @@ import type {} from "metabase-types/api";import { EnterpriseApi } from "./api";-import { tag } from "./tags";+import { invalidateTags, tag } from "./tags";export const workspaceInstanceApi = EnterpriseApi.injectEndpoints({endpoints: (builder) => ({@@ -19,6 +19,14 @@ export const workspaceInstanceApi = EnterpriseApi.injectEndpoints({response.data,providesTags: [tag("workspace")],}),+ deleteCurrentWorkspace: builder.mutation<void, void>({+ query: () => ({+ method: "DELETE",+ url: "/api/ee/workspace-instance/current",+ }),+ invalidatesTags: (_, error) =>+ invalidateTags(error, [tag("workspace"), tag("table-remapping")]),+ }),listTableRemappings: builder.query<TableRemapping[], void>({query: () => ({method: "GET",@@ -30,5 +38,8 @@ export const workspaceInstanceApi = EnterpriseApi.injectEndpoints({}),});-export const { useGetCurrentWorkspaceQuery, useListTableRemappingsQuery } =- workspaceInstanceApi;+export const {+ useGetCurrentWorkspaceQuery,+ useDeleteCurrentWorkspaceMutation,+ useListTableRemappingsQuery,+} = workspaceInstanceApi;diff --git a/enterprise/frontend/src/metabase-enterprise/api/workspace-manager.ts b/enterprise/frontend/src/metabase-enterprise/api/workspace-manager.tsindex f3ea5aeafd0..1d3b22c4653 100644--- a/enterprise/frontend/src/metabase-enterprise/api/workspace-manager.ts+++ b/enterprise/frontend/src/metabase-enterprise/api/workspace-manager.ts@@ -56,22 +56,6 @@ export const workspaceManagerApi = EnterpriseApi.injectEndpoints({invalidatesTags: (_, error, id) =>invalidateTags(error, [listTag("workspace"), idTag("workspace", id)]),}),- provisionWorkspace: builder.mutation<Workspace, WorkspaceId>({- query: (id) => ({- method: "POST",- url: `/api/ee/workspace-manager/${id}/provision`,- }),- invalidatesTags: (_, error, id) =>- invalidateTags(error, [listTag("workspace"), idTag("workspace", id)]),- }),- deprovisionWorkspace: builder.mutation<Workspace, WorkspaceId>({- query: (id) => ({- method: "POST",- url: `/api/ee/workspace-manager/${id}/deprovision`,- }),- invalidatesTags: (_, error, id) =>- invalidateTags(error, [listTag("workspace"), idTag("workspace", id)]),- }),}),});@@ -82,6 +66,4 @@ export const {useCreateWorkspaceMutation,useUpdateWorkspaceMutation,useDeleteWorkspaceMutation,- useProvisionWorkspaceMutation,- useDeprovisionWorkspaceMutation,} = workspaceManagerApi;diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/analytics.ts b/enterprise/frontend/src/metabase-enterprise/workspaces/analytics.tsindex 3082dfd4e7d..61d43d5672f 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/analytics.ts+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/analytics.ts@@ -11,3 +11,33 @@ export function trackWorkspaceCreated({target_id: workspaceId,});}++export function trackWorkspaceConfigDownloaded({+ workspaceId,+}: {+ workspaceId: WorkspaceId;+}) {+ trackSimpleEvent({+ event: "workspaces_config_downloaded",+ target_id: workspaceId,+ });+}++export function trackWorkspaceSetupButtonClicked() {+ trackSimpleEvent({+ event: "workspaces_setup_button_clicked",+ });+}++export function trackWorkspaceInstanceSetup() {+ trackSimpleEvent({+ event: "workspaces_instance_setup",+ triggered_from: "upload",+ });+}++export function trackWorkspaceInstanceLeave() {+ trackSimpleEvent({+ event: "workspaces_instance_leave",+ });+}diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/constants.ts b/enterprise/frontend/src/metabase-enterprise/workspaces/constants.tsdeleted file mode 100644index bf31193782d..00000000000--- a/enterprise/frontend/src/metabase-enterprise/workspaces/constants.ts+++ /dev/null@@ -1,3 +0,0 @@-import { isCypressActive } from "metabase/env";--export const POLLING_INTERVAL = isCypressActive ? 200 : 3000;diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/CurrentWorkspacePage.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/CurrentWorkspacePage.tsxindex f77d3506778..c58225ffb32 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/CurrentWorkspacePage.tsx+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/CurrentWorkspacePage.tsx@@ -16,6 +16,7 @@ import type {TableRemapping,} from "metabase-types/api";+import { DeleteSection } from "./DeleteSection";import { TableRemappingSection } from "./TableRemappingSection";import { getDatabasesInfo } from "./utils";@@ -89,6 +90,7 @@ function CurrentWorkspacePageBody({remappings={remappings}/>))}+ {workspace.can_write && <DeleteSection />}</Stack></PageContainer>);diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/CurrentWorkspacePage.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/CurrentWorkspacePage.unit.spec.tsxindex a86bc299833..4b766202680 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/CurrentWorkspacePage.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/CurrentWorkspacePage.unit.spec.tsx@@ -28,6 +28,7 @@ function setup({output: { schema: "ws_dev" },}),},+ can_write: true,}),} = {}) {setupDatabasesEndpoints([POSTGRES]);@@ -67,4 +68,33 @@ describe("CurrentWorkspacePage", () => {await screen.findByRole("region", { name: "Postgres" }),).toBeInTheDocument();});++ it("renders the delete section", async () => {+ setup();++ expect(await screen.findByText("Dev workspace")).toBeInTheDocument();+ expect(+ await screen.findByTestId("workspace-instance-delete-section"),+ ).toBeInTheDocument();+ });++ it("hides the delete section when can_write is false", async () => {+ setup({+ workspace: createMockCurrentWorkspace({+ name: "Dev workspace",+ databases: {+ [POSTGRES.id]: createMockCurrentWorkspaceDatabase({+ input_schemas: ["public"],+ output: { schema: "ws_dev" },+ }),+ },+ can_write: false,+ }),+ });++ expect(await screen.findByText("Dev workspace")).toBeInTheDocument();+ expect(+ screen.queryByTestId("workspace-instance-delete-section"),+ ).not.toBeInTheDocument();+ });});diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/DeleteSection/DeleteSection.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/DeleteSection/DeleteSection.tsxnew file mode 100644index 00000000000..dec300a33d9--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/DeleteSection/DeleteSection.tsx@@ -0,0 +1,53 @@+import { jt, t } from "ttag";++import { TitleSection } from "metabase/common/data-studio/components/TitleSection";+import { useConfirmation } from "metabase/common/hooks/use-confirmation";+import { Button, Code, Group, Text } from "metabase/ui";+import { useDeleteCurrentWorkspaceMutation } from "metabase-enterprise/api";++import { trackWorkspaceInstanceLeave } from "../../../analytics";++const CONFIG_FILENAME = "config.yml";++export function DeleteSection() {+ const { modalContent, show } = useConfirmation();+ const [deleteInstance] = useDeleteCurrentWorkspaceMutation();++ const handleClick = () => {+ show({+ title: t`Leave workspace?`,+ message: jt`This instance will stop remapping transform tables to the workspace's isolated schemas. The databases registered from the uploaded ${(+ <Code key="config">{CONFIG_FILENAME}</Code>+ )} will remain — you can re-enter the workspace at any time by uploading it again.`,+ confirmButtonText: t`Leave workspace`,+ confirmButtonProps: { color: "feedback-negative" },+ onConfirm: async () => {+ await deleteInstance().unwrap();+ trackWorkspaceInstanceLeave();+ },+ });+ };++ return (+ <>+ <TitleSection+ data-testid="workspace-instance-delete-section"+ label={t`Leave workspace`}+ >+ <Group p="lg" justify="space-between" align="center">+ <Text maw="40rem">+ {t`Stop remapping transform tables on this instance.`}+ </Text>+ <Button+ color="feedback-negative"+ variant="filled"+ onClick={handleClick}+ >+ {t`Leave workspace`}+ </Button>+ </Group>+ </TitleSection>+ {modalContent}+ </>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/DeleteSection/DeleteSection.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/DeleteSection/DeleteSection.unit.spec.tsxnew file mode 100644index 00000000000..30b061b1e87--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/DeleteSection/DeleteSection.unit.spec.tsx@@ -0,0 +1,63 @@+import userEvent from "@testing-library/user-event";+import fetchMock from "fetch-mock";++import { setupDeleteCurrentWorkspaceEndpoint } from "__support__/server-mocks";+import { renderWithProviders, screen, waitFor, within } from "__support__/ui";+import { Route } from "metabase/router";++import { DeleteSection } from "./DeleteSection";++const { trackSimpleEvent } = jest.requireMock("metabase/analytics");++function setup() {+ setupDeleteCurrentWorkspaceEndpoint();++ renderWithProviders(<Route path="*" element={<DeleteSection />} />, {+ withRouter: true,+ });+}++describe("DeleteSection", () => {+ beforeEach(() => {+ trackSimpleEvent.mockClear();+ });++ it("calls DELETE /api/ee/workspace-instance/current after confirming", async () => {+ setup();++ await userEvent.click(+ screen.getByRole("button", { name: "Leave workspace" }),+ );+ const dialog = await screen.findByRole("dialog");+ await userEvent.click(+ within(dialog).getByRole("button", { name: "Leave workspace" }),+ );++ await waitFor(() => {+ expect(+ fetchMock.callHistory.called(+ "path:/api/ee/workspace-instance/current",+ { method: "DELETE" },+ ),+ ).toBe(true);+ });+ });++ it("tracks an analytics event when the instance is torn down", async () => {+ setup();++ await userEvent.click(+ screen.getByRole("button", { name: "Leave workspace" }),+ );+ const dialog = await screen.findByRole("dialog");+ await userEvent.click(+ within(dialog).getByRole("button", { name: "Leave workspace" }),+ );++ await waitFor(() =>+ expect(trackSimpleEvent).toHaveBeenCalledWith({+ event: "workspaces_instance_leave",+ }),+ );+ });+});diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/DeleteSection/index.ts b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/DeleteSection/index.tsnew file mode 100644index 00000000000..50566513525--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/CurrentWorkspacePage/DeleteSection/index.ts@@ -0,0 +1 @@+export * from "./DeleteSection";diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/DeleteWorkspaceModal/DeleteWorkspaceModal.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/DeleteWorkspaceModal/DeleteWorkspaceModal.tsxnew file mode 100644index 00000000000..9ddb43b4fb1--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/DeleteWorkspaceModal/DeleteWorkspaceModal.tsx@@ -0,0 +1,143 @@+import { t } from "ttag";++import { DelayedLoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper/DelayedLoadingAndErrorWrapper";+import {+ Form,+ FormErrorMessage,+ FormProvider,+ FormSubmitButton,+} from "metabase/forms";+import {+ Button,+ FocusTrap,+ Group,+ List,+ Modal,+ Stack,+ Text,+} from "metabase/ui";+import {+ useDeleteWorkspaceMutation,+ useGetWorkspaceQuery,+} from "metabase-enterprise/api";+import type { Workspace, WorkspaceId } from "metabase-types/api";++import { getWorkspaceDatabaseName, isPending } from "../../../utils";++export type DeleteWorkspaceModalProps = {+ workspaceId: WorkspaceId;+ onDelete: () => void;+ onClose: () => void;+};++export function DeleteWorkspaceModal({+ workspaceId,+ onDelete,+ onClose,+}: DeleteWorkspaceModalProps) {+ return (+ <Modal+ title={t`Delete this workspace?`}+ opened+ padding="xl"+ onClose={onClose}+ >+ <FocusTrap.InitialFocus />+ <DeleteWorkspaceLoader+ workspaceId={workspaceId}+ onDelete={onDelete}+ onClose={onClose}+ />+ </Modal>+ );+}++type DeleteWorkspaceLoaderProps = {+ workspaceId: WorkspaceId;+ onDelete: () => void;+ onClose: () => void;+};++function DeleteWorkspaceLoader({+ workspaceId,+ onDelete,+ onClose,+}: DeleteWorkspaceLoaderProps) {+ // Fetch the workspace on open so the pending-databases warning reflects+ // fresh statuses, not the possibly stale list-page snapshot.+ const {+ data: workspace,+ isLoading,+ error,+ } = useGetWorkspaceQuery(workspaceId);++ if (isLoading || error != null || workspace == null) {+ return <DelayedLoadingAndErrorWrapper loading={isLoading} error={error} />;+ }++ return (+ <DeleteWorkspaceForm+ workspace={workspace}+ onDelete={onDelete}+ onClose={onClose}+ />+ );+}++type DeleteWorkspaceFormProps = {+ workspace: Workspace;+ onDelete: () => void;+ onClose: () => void;+};++function DeleteWorkspaceForm({+ workspace,+ onDelete,+ onClose,+}: DeleteWorkspaceFormProps) {+ const [deleteWorkspace] = useDeleteWorkspaceMutation();+ const databases = workspace.databases ?? [];+ const pendingDatabases = databases.filter(isPending);+ const hasPendingDatabases = pendingDatabases.length > 0;++ const handleSubmit = async () => {+ await deleteWorkspace(workspace.id).unwrap();+ onDelete();+ };++ return (+ <FormProvider initialValues={{}} onSubmit={handleSubmit}>+ <Form>+ <Stack gap="lg">+ {hasPendingDatabases ? (+ <Stack gap="sm">+ <Text>+ {t`Some of this workspace's databases are still being set up or torn down. Deleting will wait for that work to finish, then remove the workspace along with its temporary database users and schemas:`}+ </Text>+ <List>+ {pendingDatabases.map((workspaceDatabase) => (+ <List.Item key={workspaceDatabase.database_id}>+ {getWorkspaceDatabaseName(workspaceDatabase)}+ </List.Item>+ ))}+ </List>+ </Stack>+ ) : (+ <Text>+ {t`This will delete the workspace as well as the temporary database users and schemas that were created for this workspace. This can't be undone.`}+ </Text>+ )}+ <FormErrorMessage />+ <Group justify="flex-end">+ <Button onClick={onClose}>{t`Cancel`}</Button>+ <FormSubmitButton+ label={t`Delete workspace`}+ variant="filled"+ color="feedback-negative"+ />+ </Group>+ </Stack>+ </Form>+ </FormProvider>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/DeleteWorkspaceModal/DeleteWorkspaceModal.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/DeleteWorkspaceModal/DeleteWorkspaceModal.unit.spec.tsxnew file mode 100644index 00000000000..c75c618a4ed--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/DeleteWorkspaceModal/DeleteWorkspaceModal.unit.spec.tsx@@ -0,0 +1,119 @@+import userEvent from "@testing-library/user-event";+import fetchMock from "fetch-mock";++import {+ setupDeleteWorkspaceEndpoint,+ setupDeleteWorkspaceEndpointError,+ setupGetWorkspaceEndpoint,+} from "__support__/server-mocks";+import { renderWithProviders, screen, waitFor } from "__support__/ui";+import { UndoListing } from "metabase/common/components/UndoListing";+import type { WorkspaceDatabase } from "metabase-types/api";+import {+ createMockDatabase,+ createMockWorkspace,+ createMockWorkspaceDatabase,+} from "metabase-types/api/mocks";++import { DeleteWorkspaceModal } from "./DeleteWorkspaceModal";++function setup({+ withError = false,+ databases = [],+}: {+ withError?: boolean;+ databases?: WorkspaceDatabase[];+} = {}) {+ const workspace = createMockWorkspace({+ id: 1,+ name: "My workspace",+ databases,+ });+ setupGetWorkspaceEndpoint(workspace);++ if (withError) {+ setupDeleteWorkspaceEndpointError(workspace.id);+ } else {+ setupDeleteWorkspaceEndpoint(workspace.id);+ }++ const onDelete = jest.fn();+ const onClose = jest.fn();++ renderWithProviders(+ <>+ <DeleteWorkspaceModal+ workspaceId={workspace.id}+ onDelete={onDelete}+ onClose={onClose}+ />+ <UndoListing />+ </>,+ );++ return { workspace, onDelete, onClose };+}++async function clickDelete() {+ const button = await screen.findByRole("button", {+ name: "Delete workspace",+ });+ await waitFor(() => expect(button).toBeEnabled());+ await userEvent.click(button);+}++describe("DeleteWorkspaceModal", () => {+ it("should delete the workspace and call the callback when the confirm button is clicked", async () => {+ const { onDelete } = setup();++ await clickDelete();++ await waitFor(() => {+ expect(+ fetchMock.callHistory.called("path:/api/ee/workspace-manager/1", {+ method: "DELETE",+ }),+ ).toBe(true);+ });+ await waitFor(() => expect(onDelete).toHaveBeenCalled());+ });++ it("should list pending databases and delete when confirmed", async () => {+ const { onDelete } = setup({+ databases: [+ createMockWorkspaceDatabase({+ database_id: 7,+ status: "provisioning",+ database: createMockDatabase({ id: 7, name: "Pending DB" }),+ }),+ ],+ });++ expect(+ await screen.findByText(/still being set up or torn down/i),+ ).toBeInTheDocument();+ expect(screen.getByText("Pending DB")).toBeInTheDocument();++ await clickDelete();++ await waitFor(() => {+ expect(+ fetchMock.callHistory.called("path:/api/ee/workspace-manager/1", {+ method: "DELETE",+ }),+ ).toBe(true);+ });+ await waitFor(() => expect(onDelete).toHaveBeenCalled());+ });++ it("should show an error message and not call the callback when the request fails", async () => {+ const { onDelete } = setup({ withError: true });++ await clickDelete();++ expect(await screen.findByRole("alert")).toHaveTextContent(+ "An error occurred",+ );+ expect(onDelete).not.toHaveBeenCalled();+ });+});diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/DeleteWorkspaceModal/index.ts b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/DeleteWorkspaceModal/index.tsnew file mode 100644index 00000000000..a3307e8c5c3--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/DeleteWorkspaceModal/index.ts@@ -0,0 +1 @@+export * from "./DeleteWorkspaceModal";diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceButton/NewWorkspaceButton.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceButton/NewWorkspaceButton.tsxindex c8070890504..09376acc35b 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceButton/NewWorkspaceButton.tsx+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceButton/NewWorkspaceButton.tsx@@ -1,11 +1,13 @@import { useDisclosure } from "@mantine/hooks";import { t } from "ttag";-import { useSetting } from "metabase/common/hooks";+import {+ hasFeature,+ hasWorkspacesEnabled,+} from "metabase/common/utils/database";import { Button, FixedSizeIcon, Tooltip } from "metabase/ui";import type { Database } from "metabase-types/api";-import { getEligibleDatabases } from "../../../utils";import { NewWorkspaceModal } from "../NewWorkspaceModal";export type NewWorkspaceButtonProps = {@@ -18,28 +20,25 @@ export function NewWorkspaceButton({primary,}: NewWorkspaceButtonProps) {const [opened, { open, close }] = useDisclosure(false);- const isRemoteSyncEnabled = useSetting("remote-sync-enabled") ?? false;- const eligibleDatabases = getEligibleDatabases(databases);+ const eligibleDatabases = databases.filter(+ (database) =>+ hasFeature(database, "workspace") && hasWorkspacesEnabled(database),+ );const hasEligibleDatabase = eligibleDatabases.length > 0;- const canCreateWorkspace = hasEligibleDatabase && isRemoteSyncEnabled;return (<><Tooltip- label={- hasEligibleDatabase- ? t`You need to set up remote sync.`- : t`You need to enable workspaces on at least one database.`- }- disabled={canCreateWorkspace}+ label={t`You need to enable workspaces on at least one database.`}+ disabled={hasEligibleDatabase}><Buttonvariant={primary ? "filled" : "default"}leftSection={primary ? undefined : <FixedSizeIcon name="add" aria-hidden />}- disabled={!canCreateWorkspace}+ disabled={!hasEligibleDatabase}onClick={open}>{primary ? t`Create a workspace` : t`New`}diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceButton/NewWorkspaceButton.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceButton/NewWorkspaceButton.unit.spec.tsxindex 7578af005eb..39eb7975856 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceButton/NewWorkspaceButton.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceButton/NewWorkspaceButton.unit.spec.tsx@@ -1,35 +1,29 @@import userEvent from "@testing-library/user-event";import { renderWithProviders, screen } from "__support__/ui";-import { createMockSettingsState } from "metabase/redux/store/mocks";import type { Database } from "metabase-types/api";import { createMockDatabase } from "metabase-types/api/mocks";import { NewWorkspaceButton } from "./NewWorkspaceButton";-const ELIGIBLE_DATABASE = createMockDatabase({- features: ["workspace"],- settings: { "database-enable-workspaces": true },-});-type SetupOpts = {databases?: Database[];- isRemoteSyncEnabled?: boolean;};-function setup({ databases = [], isRemoteSyncEnabled = true }: SetupOpts = {}) {- renderWithProviders(<NewWorkspaceButton databases={databases} />, {- storeInitialState: {- settings: createMockSettingsState({- "remote-sync-enabled": isRemoteSyncEnabled,- }),- },- });+function setup({ databases = [] }: SetupOpts = {}) {+ renderWithProviders(<NewWorkspaceButton databases={databases} />);}describe("NewWorkspaceButton", () => {it("opens the New workspace modal when clicked", async () => {- setup({ databases: [ELIGIBLE_DATABASE] });+ setup({+ databases: [+ createMockDatabase({+ features: ["workspace"],+ settings: { "database-enable-workspaces": true },+ }),+ ],+ });await userEvent.click(screen.getByRole("button", { name: /New/ }));@@ -43,10 +37,4 @@ describe("NewWorkspaceButton", () => {expect(screen.getByRole("button", { name: /New/ })).toBeDisabled();});-- it("is disabled when remote sync is not set up", () => {- setup({ databases: [ELIGIBLE_DATABASE], isRemoteSyncEnabled: false });-- expect(screen.getByRole("button", { name: /New/ })).toBeDisabled();- });});diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceModal/NewWorkspaceModal.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceModal/NewWorkspaceModal.tsxindex 7d3a2f05d49..1659e893adb 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceModal/NewWorkspaceModal.tsx+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceModal/NewWorkspaceModal.tsx@@ -1,9 +1,7 @@-import { useFormikContext } from "formik";-import { type ChangeEvent, useState } from "react";-import slugg from "slugg";import { t } from "ttag";import * as Yup from "yup";+import { useToast } from "metabase/common/hooks/use-toast";import {Form,FormCheckboxGroup,@@ -17,11 +15,11 @@ import * as Errors from "metabase/utils/errors";import {useCreateWorkspaceMutation,useLazyListWorkspacesQuery,- useProvisionWorkspaceMutation,} from "metabase-enterprise/api";import type { Database, Workspace } from "metabase-types/api";import { trackWorkspaceCreated } from "../../../analytics";+import { getProvisioningFailureMessage, isUnprovisioned } from "../../../utils";type NewWorkspaceModalProps = {databases: Database[];@@ -54,23 +52,18 @@ export function NewWorkspaceModal({type NewWorkspaceFormValues = {name: string;- target_branch: string;database_ids: string[];};const NEW_WORKSPACE_SCHEMA = Yup.object({name: Yup.string().required(Errors.required),- target_branch: Yup.string().required(Errors.required),database_ids: Yup.array().of(Yup.string()).min(1, Errors.required),});-function getInitialValues(databases: Database[]): NewWorkspaceFormValues {- return {- name: "",- target_branch: "",- database_ids: databases.length === 1 ? [String(databases[0].id)] : [],- };-}+const INITIAL_VALUES: NewWorkspaceFormValues = {+ name: "",+ database_ids: [],+};type NewWorkspaceFormProps = {databases: Database[];@@ -84,34 +77,58 @@ function NewWorkspaceForm({onClose,}: NewWorkspaceFormProps) {const [createWorkspace] = useCreateWorkspaceMutation();- const [provisionWorkspace] = useProvisionWorkspaceMutation();const [fetchWorkspaces] = useLazyListWorkspacesQuery();+ const [sendToast] = useToast();const handleSubmit = async ({name,- target_branch,database_ids,}: NewWorkspaceFormValues) => {const workspace = await createWorkspace({name,- target_branch,database_ids: database_ids.map(Number),}).unwrap();trackWorkspaceCreated({ workspaceId: workspace.id });- await provisionWorkspace(workspace.id);++ const workspaceDatabases = workspace.databases ?? [];+ if (workspaceDatabases.some(isUnprovisioned)) {+ sendToast({+ message: getProvisioningFailureMessage(),+ icon: "warning",+ toastColor: "feedback-negative",+ timeout: null,+ });+ }+await fetchWorkspaces();onCreate(workspace);};return (<FormProvider- initialValues={getInitialValues(databases)}+ initialValues={INITIAL_VALUES}validationSchema={NEW_WORKSPACE_SCHEMA}onSubmit={handleSubmit}><Form><Stack gap="lg">- <NewWorkspaceFormFields databases={databases} />+ <FormTextInput+ name="name"+ label={t`Name`}+ placeholder={t`My workspace`}+ data-autofocus+ />+ <FormCheckboxGroup name="database_ids" label={t`Databases`}>+ <Stack gap="sm" mt="sm">+ {databases.map((database) => (+ <Checkbox+ key={database.id}+ value={String(database.id)}+ label={database.name}+ />+ ))}+ </Stack>+ </FormCheckboxGroup><FormErrorMessage /><Group justify="flex-end"><Button onClick={onClose}>{t`Cancel`}</Button>@@ -122,47 +139,3 @@ function NewWorkspaceForm({</FormProvider>);}--type NewWorkspaceFormFieldsProps = {- databases: Database[];-};--function NewWorkspaceFormFields({ databases }: NewWorkspaceFormFieldsProps) {- const { setFieldValue } = useFormikContext<NewWorkspaceFormValues>();- const [isBranchTouched, setIsBranchTouched] = useState(false);-- const handleNameChange = (event: ChangeEvent<HTMLInputElement>) => {- if (!isBranchTouched) {- setFieldValue("target_branch", slugg(event.target.value));- }- };-- return (- <>- <FormTextInput- name="name"- label={t`Name`}- placeholder={t`My workspace`}- data-autofocus- onChange={handleNameChange}- />- <FormTextInput- name="target_branch"- label={t`Branch`}- placeholder={t`my-workspace`}- onChange={() => setIsBranchTouched(true)}- />- <FormCheckboxGroup name="database_ids" label={t`Databases`}>- <Stack gap="sm" mt="sm">- {databases.map((database) => (- <Checkbox- key={database.id}- value={String(database.id)}- label={database.name}- />- ))}- </Stack>- </FormCheckboxGroup>- </>- );-}diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceModal/NewWorkspaceModal.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceModal/NewWorkspaceModal.unit.spec.tsxindex 0bc0b5f2097..be957354703 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceModal/NewWorkspaceModal.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/NewWorkspaceModal/NewWorkspaceModal.unit.spec.tsx@@ -1,17 +1,16 @@import userEvent from "@testing-library/user-event";-import fetchMock from "fetch-mock";import {setupCreateWorkspaceEndpoint,setupListWorkspacesEndpoint,- setupProvisionWorkspaceEndpoint,} from "__support__/server-mocks";import { renderWithProviders, screen, waitFor } from "__support__/ui";import { UndoListing } from "metabase/common/components/UndoListing";-import type { Database, Workspace } from "metabase-types/api";+import type { Workspace } from "metabase-types/api";import {createMockDatabase,createMockWorkspace,+ createMockWorkspaceDatabase,} from "metabase-types/api/mocks";import { NewWorkspaceModal } from "./NewWorkspaceModal";@@ -20,27 +19,23 @@ const { trackSimpleEvent } = jest.requireMock("metabase/analytics");type SetupOpts = {createdWorkspace?: Workspace;- databases?: Database[];};const DATABASE = createMockDatabase({ id: 10, name: "Postgres" });-const DATABASE_2 = createMockDatabase({ id: 20, name: "MySQL" });function setup({createdWorkspace = createMockWorkspace({ name: "Brand new workspace" }),- databases = [DATABASE, DATABASE_2],}: SetupOpts = {}) {const onCreate = jest.fn();const onClose = jest.fn();setupCreateWorkspaceEndpoint(createdWorkspace);- setupProvisionWorkspaceEndpoint(createdWorkspace);setupListWorkspacesEndpoint([createdWorkspace]);renderWithProviders(<><NewWorkspaceModal- databases={databases}+ databases={[DATABASE]}openedonCreate={onCreate}onClose={onClose}@@ -69,83 +64,32 @@ describe("NewWorkspaceModal", () => {await waitFor(() =>expect(onCreate).toHaveBeenCalledWith(createdWorkspace),);- expect(- fetchMock.callHistory.called(- `path:/api/ee/workspace-manager/${createdWorkspace.id}/provision`,- { method: "POST" },- ),- ).toBe(true);- });-- it("fills the branch with the sluggified name until manually edited", async () => {- setup();-- await userEvent.type(screen.getByLabelText("Name"), "Brand New Workspace");- expect(screen.getByLabelText("Branch")).toHaveValue("brand-new-workspace");- });-- it("keeps a manually edited branch when the name changes", async () => {- setup();-- await userEvent.type(screen.getByLabelText("Branch"), "custom-branch");- await userEvent.type(screen.getByLabelText("Name"), "Brand new workspace");-- expect(screen.getByLabelText("Branch")).toHaveValue("custom-branch");});- it("sends the branch when creating a workspace", async () => {- const { onCreate } = setup();-- await userEvent.type(screen.getByLabelText("Name"), "Brand new workspace");- await userEvent.click(screen.getByRole("checkbox", { name: "Postgres" }));- await userEvent.click(- screen.getByRole("button", { name: "Create workspace" }),- );-- await waitFor(() => expect(onCreate).toHaveBeenCalled());- const call = fetchMock.callHistory.lastCall(- "path:/api/ee/workspace-manager",- { method: "POST" },- );- expect(await call?.request?.json()).toMatchObject({- target_branch: "brand-new-workspace",+ it("warns when some databases could not be provisioned", async () => {+ const { onCreate } = setup({+ createdWorkspace: createMockWorkspace({+ name: "Brand new workspace",+ databases: [+ createMockWorkspaceDatabase({+ database_id: 10,+ status: "unprovisioned",+ database: createMockDatabase({ id: 10, name: "Postgres" }),+ }),+ ],+ }),});- });-- it("requires a branch", async () => {- const { onCreate } = setup();await userEvent.type(screen.getByLabelText("Name"), "Brand new workspace");- await userEvent.clear(screen.getByLabelText("Branch"));await userEvent.click(screen.getByRole("checkbox", { name: "Postgres" }));await userEvent.click(screen.getByRole("button", { name: "Create workspace" }),);- expect(onCreate).not.toHaveBeenCalled();- });-- it("preselects the database when it is the only one available", async () => {- const { onCreate, createdWorkspace } = setup({ databases: [DATABASE] });-- expect(screen.getByRole("checkbox", { name: "Postgres" })).toBeChecked();- await userEvent.type(screen.getByLabelText("Name"), "Brand new workspace");- await userEvent.click(- screen.getByRole("button", { name: "Create workspace" }),- );-- await waitFor(() =>- expect(onCreate).toHaveBeenCalledWith(createdWorkspace),- );- });-- it("preselects nothing when several databases are available", () => {- setup();-expect(- screen.getByRole("checkbox", { name: "Postgres" }),- ).not.toBeChecked();- expect(screen.getByRole("checkbox", { name: "MySQL" })).not.toBeChecked();+ await screen.findByText("Failed to provision the workspace."),+ ).toBeInTheDocument();+ await waitFor(() => expect(onCreate).toHaveBeenCalled());});it("requires at least one database", async () => {diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/SetupWorkspaceModal/SetupWorkspaceModal.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/SetupWorkspaceModal/SetupWorkspaceModal.tsxnew file mode 100644index 00000000000..377849a1ee0--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/SetupWorkspaceModal/SetupWorkspaceModal.tsx@@ -0,0 +1,108 @@+import { jt, t } from "ttag";++import {+ Form,+ FormErrorMessage,+ FormFileInput,+ FormProvider,+ FormSubmitButton,+} from "metabase/forms";+import { useDispatch } from "metabase/redux";+import { push } from "metabase/router";+import {+ Button,+ Code,+ FixedSizeIcon,+ FocusTrap,+ Group,+ Modal,+ Stack,+ Text,+} from "metabase/ui";+import * as Urls from "metabase/urls";+import { useApplyAdvancedConfigMutation } from "metabase-enterprise/api";++import { trackWorkspaceInstanceSetup } from "../../../analytics";++const CONFIG_FILENAME = "config.yml";++type SetupWorkspaceModalProps = {+ opened: boolean;+ onClose: () => void;+};++export function SetupWorkspaceModal({+ opened,+ onClose,+}: SetupWorkspaceModalProps) {+ return (+ <Modal+ title={t`Set up a workspace`}+ opened={opened}+ padding="xl"+ onClose={onClose}+ >+ <FocusTrap.InitialFocus />+ <SetupWorkspaceForm onClose={onClose} />+ </Modal>+ );+}++type SetupWorkspaceFormValues = {+ config: File | null;+};++const INITIAL_VALUES: SetupWorkspaceFormValues = {+ config: null,+};++type SetupWorkspaceFormProps = {+ onClose: () => void;+};++function SetupWorkspaceForm({ onClose }: SetupWorkspaceFormProps) {+ const dispatch = useDispatch();+ const [applyConfig] = useApplyAdvancedConfigMutation();++ const handleSubmit = async ({ config }: SetupWorkspaceFormValues) => {+ if (!config) {+ return;+ }+ await applyConfig({ config }).unwrap();+ trackWorkspaceInstanceSetup();+ onClose();+ dispatch(push(Urls.workspaces()));+ };++ return (+ <FormProvider initialValues={INITIAL_VALUES} onSubmit={handleSubmit}>+ {({ values }) => (+ <Form>+ <Stack gap="lg">+ <Text c="text-secondary">+ {jt`Upload the ${(+ <Code key="config">{CONFIG_FILENAME}</Code>+ )} you downloaded from a workspace on your production instance.`}+ </Text>+ <FormFileInput+ name="config"+ label={t`Config file`}+ placeholder={t`Choose a ${CONFIG_FILENAME} file`}+ accept=".yml,.yaml,application/yaml,text/yaml"+ leftSection={<FixedSizeIcon name="attachment" aria-hidden />}+ />+ <FormErrorMessage />+ <Group justify="flex-end">+ <Button onClick={onClose}>{t`Cancel`}</Button>+ <FormSubmitButton+ label={t`Set up`}+ variant="filled"+ disabled={values.config == null}+ />+ </Group>+ </Stack>+ </Form>+ )}+ </FormProvider>+ );+}diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/SetupWorkspaceModal/index.ts b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/SetupWorkspaceModal/index.tsnew file mode 100644index 00000000000..66392812b9e--- /dev/null+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/SetupWorkspaceModal/index.ts@@ -0,0 +1 @@+export { SetupWorkspaceModal } from "./SetupWorkspaceModal";diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/StatusDetailsModal/StatusDetailsModal.module.css b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/StatusDetailsModal/StatusDetailsModal.module.cssdeleted file mode 100644index 0abd65746e0..00000000000--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/StatusDetailsModal/StatusDetailsModal.module.css+++ /dev/null@@ -1,5 +0,0 @@-.codeContainer {- border: var(--border-size) var(--border-style) var(--mb-color-border-neutral);- border-radius: var(--default-border-radius);- overflow: hidden;-}diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/StatusDetailsModal/StatusDetailsModal.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/StatusDetailsModal/StatusDetailsModal.tsxdeleted file mode 100644index f5735e27aac..00000000000--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/StatusDetailsModal/StatusDetailsModal.tsx+++ /dev/null@@ -1,33 +0,0 @@-import { CodeEditor } from "metabase/common/components/CodeEditor";-import { Modal } from "metabase/ui";-import type { Workspace } from "metabase-types/api";--import { getStatusMessage } from "../../../utils";--import S from "./StatusDetailsModal.module.css";--export type StatusDetailsModalProps = {- workspace: Workspace;- opened: boolean;- onClose: () => void;-};--export function StatusDetailsModal({- workspace,- opened,- onClose,-}: StatusDetailsModalProps) {- return (- <Modal- title={getStatusMessage(workspace.status)}- opened={opened}- padding="xl"- size="lg"- onClose={onClose}- >- <div className={S.codeContainer}>- <CodeEditor value={workspace.status_details ?? ""} readOnly />- </div>- </Modal>- );-}diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/StatusDetailsModal/index.ts b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/StatusDetailsModal/index.tsdeleted file mode 100644index d2adc285d59..00000000000--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/StatusDetailsModal/index.ts+++ /dev/null@@ -1 +0,0 @@-export { StatusDetailsModal } from "./StatusDetailsModal";diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceEmptyState/WorkspaceEmptyState.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceEmptyState/WorkspaceEmptyState.tsxindex 9ed5b50d078..69df2534384 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceEmptyState/WorkspaceEmptyState.tsx+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceEmptyState/WorkspaceEmptyState.tsx@@ -1,12 +1,13 @@-import { jt, t } from "ttag";+import { useDisclosure } from "@mantine/hooks";+import { t } from "ttag";-import { useDocsUrl, useSetting } from "metabase/common/hooks";+import { useDocsUrl } from "metabase/common/hooks";import { useSelector } from "metabase/redux";import { Link } from "metabase/router";import { getApplicationName } from "metabase/selectors/whitelabel";import {- Anchor,Box,+ Button,Card,Divider,FixedSizeIcon,@@ -15,11 +16,11 @@ import {Text,Title,} from "metabase/ui";-import * as Urls from "metabase/urls";import type { Database } from "metabase-types/api";-import { getEligibleDatabases } from "../../../utils";+import { trackWorkspaceSetupButtonClicked } from "../../../analytics";import { NewWorkspaceButton } from "../NewWorkspaceButton";+import { SetupWorkspaceModal } from "../SetupWorkspaceModal";import S from "./WorkspaceEmptyState.module.css";@@ -28,6 +29,8 @@ export type WorkspaceEmptyStateProps = {};export function WorkspaceEmptyState({ databases }: WorkspaceEmptyStateProps) {+ const [isSetupOpen, { open: openSetup, close: closeSetup }] =+ useDisclosure(false);const applicationName = useSelector(getApplicationName);const { url: fileBasedDevDocsUrl, showMetabaseLinks: showFileBasedDevLink } =@@ -35,82 +38,60 @@ export function WorkspaceEmptyState({ databases }: WorkspaceEmptyStateProps) {const { url: remoteSyncDocsUrl, showMetabaseLinks: showRemoteSyncLink } =useDocsUrl("installation-and-operation/remote-sync");+ const handleSetupClick = () => {+ trackWorkspaceSetupButtonClicked();+ openSetup();+ };+return (- <Card p="xl" maw="40rem" mx="auto" shadow="none" withBorder>- <Box p="md">- <Title- order={3}- mb="sm"- >{t`Use Workspaces to develop your semantic layer safely`}</Title>- <Text mb="lg">- {t`While in a workspace, ${applicationName} will remap tables created by transforms to an isolated schema, letting you test and build on top of these tables. When you're ready, use remote sync to pull your changes into your production ${applicationName}.`}- </Text>- <CreateWorkspaceSection databases={databases} />- {(showFileBasedDevLink || showRemoteSyncLink) && (- <>- <Divider my="xl" />- <Group gap="sm" align="stretch">- {showFileBasedDevLink && (- <DocsLink- title={t`Agent-driven development`}- description={t`How to use the CLI to develop content locally.`}- link={fileBasedDevDocsUrl}- />- )}- {showRemoteSyncLink && (- <DocsLink- title={t`Using remote sync`}- description={t`How to sync and review ${applicationName} content with git.`}- link={remoteSyncDocsUrl}- />- )}- </Group>- </>- )}- </Box>- </Card>+ <>+ <Card p="xl" maw="40rem" mx="auto" shadow="none" withBorder>+ <Box p="md">+ <Title+ order={3}+ mb="sm"+ >{t`Use Workspaces to develop your semantic layer safely`}</Title>+ <Text mb="md">+ {t`While in a workspace, ${applicationName} will remap tables created by transforms to an isolated schema, letting you test and build on top of these tables. When you're ready, use remote sync to pull your changes into your production ${applicationName}.`}+ </Text>+ <Text mb="lg">+ {t`If this is your production instance, create and download a workspace config here to use in a development instance.`}{" "}+ {t`If you're using this ${applicationName} instance for development, you can upload a workspace config file to put this instance into that workspace.`}+ </Text>+ <Group gap="md">+ <NewWorkspaceButton databases={databases} primary />+ <Button variant="default" onClick={handleSetupClick}>+ {t`Upload a workspace config`}+ </Button>+ </Group>+ {(showFileBasedDevLink || showRemoteSyncLink) && (+ <>+ <Divider my="xl" />+ <Group gap="sm" align="stretch">+ {showFileBasedDevLink && (+ <DocsLink+ title={t`Agent-driven development`}+ description={t`How to use the CLI to develop content locally.`}+ link={fileBasedDevDocsUrl}+ />+ )}+ {showRemoteSyncLink && (+ <DocsLink+ title={t`Using remote sync`}+ description={t`How to sync and review ${applicationName} content with git.`}+ link={remoteSyncDocsUrl}+ />+ )}+ </Group>+ </>+ )}+ </Box>+ </Card>+ <SetupWorkspaceModal opened={isSetupOpen} onClose={closeSetup} />+ </>);}-type CreateWorkspaceSectionProps = {- databases: Database[];-};--function CreateWorkspaceSection({ databases }: CreateWorkspaceSectionProps) {- const isRemoteSyncEnabled = useSetting("remote-sync-enabled");- const hasEligibleDatabase = getEligibleDatabases(databases).length > 0;-- if (!hasEligibleDatabase) {- return (- <Text>- {jt`To create a workspace, you need to enable workspaces on at least one database in ${(- <Anchor- key="link"- component={Link}- to={Urls.viewDatabases()}- >{t`admin settings`}</Anchor>- )}.`}- </Text>- );- }-- if (!isRemoteSyncEnabled) {- return (- <Text>- {jt`To create a workspace, you need to set up remote sync in ${(- <Anchor- key="link"- component={Link}- to={Urls.remoteSyncSettings()}- >{t`admin settings`}</Anchor>- )}.`}- </Text>- );- }-- return <NewWorkspaceButton databases={databases} primary />;-}-type DocsLinkProps = {title: string;description: string;diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceEmptyState/WorkspaceEmptyState.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceEmptyState/WorkspaceEmptyState.unit.spec.tsxindex 9fafd51b68e..faf8465ab41 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceEmptyState/WorkspaceEmptyState.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceEmptyState/WorkspaceEmptyState.unit.spec.tsx@@ -1,80 +1,32 @@+import { setupApplyAdvancedConfigEndpoint } from "__support__/server-mocks";import { renderWithProviders, screen } from "__support__/ui";import { createMockSettingsState } from "metabase/redux/store/mocks";import { Route } from "metabase/router";-import type { Database } from "metabase-types/api";-import { createMockDatabase } from "metabase-types/api/mocks";import { WorkspaceEmptyState } from "./WorkspaceEmptyState";-const ELIGIBLE_DATABASE = createMockDatabase({- features: ["workspace"],- settings: { "database-enable-workspaces": true },-});--type SetupOpts = {- databases?: Database[];- isRemoteSyncEnabled?: boolean;-};--function setup({ databases = [], isRemoteSyncEnabled = true }: SetupOpts = {}) {+function setup() {+ setupApplyAdvancedConfigEndpoint();renderWithProviders(- <Route path="*" element={<WorkspaceEmptyState databases={databases} />} />,+ <Route path="*" element={<WorkspaceEmptyState databases={[]} />} />,{withRouter: true,storeInitialState: {- settings: createMockSettingsState({- "show-metabase-links": true,- "remote-sync-enabled": isRemoteSyncEnabled,- }),+ settings: createMockSettingsState({ "show-metabase-links": true }),},},);}describe("WorkspaceEmptyState", () => {- it("renders the create workspace button when there is an eligible database", () => {- setup({ databases: [ELIGIBLE_DATABASE] });+ it("renders the create workspace button", () => {+ setup();expect(screen.getByRole("button", { name: "Create a workspace" }),).toBeInTheDocument();});- it("links to the databases admin page when no database is eligible", () => {- setup({ databases: [createMockDatabase()] });-- expect(- screen.queryByRole("button", { name: "Create a workspace" }),- ).not.toBeInTheDocument();- expect(- screen.getByText(/enable workspaces on at least one database/),- ).toBeInTheDocument();- expect(- screen.getByRole("link", { name: "admin settings" }),- ).toHaveAttribute("href", "/admin/databases");- });-- it("links to the remote sync settings when remote sync is not set up", () => {- setup({ databases: [ELIGIBLE_DATABASE], isRemoteSyncEnabled: false });-- expect(- screen.queryByRole("button", { name: "Create a workspace" }),- ).not.toBeInTheDocument();- expect(screen.getByText(/set up remote sync/)).toBeInTheDocument();- expect(- screen.getByRole("link", { name: "admin settings" }),- ).toHaveAttribute("href", "/admin/settings/remote-sync");- });-- it("shows the database error first when both are missing", () => {- setup({ databases: [], isRemoteSyncEnabled: false });-- expect(- screen.getByText(/enable workspaces on at least one database/),- ).toBeInTheDocument();- expect(screen.queryByText(/set up remote sync/)).not.toBeInTheDocument();- });-it("renders docs link buttons", () => {setup();@@ -86,11 +38,11 @@ describe("WorkspaceEmptyState", () => {).toBeInTheDocument();});- it("does not offer a config upload", () => {+ it("shows the upload-config button", () => {setup();expect(- screen.queryByRole("button", { name: /upload/i }),- ).not.toBeInTheDocument();+ screen.getByRole("button", { name: "Upload a workspace config" }),+ ).toBeInTheDocument();});});diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceItem/WorkspaceItem.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceItem/WorkspaceItem.tsxindex f776ab002c1..4bb84881e54 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceItem/WorkspaceItem.tsx+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceItem/WorkspaceItem.tsx@@ -1,38 +1,29 @@import { useDisclosure } from "@mantine/hooks";import { t } from "ttag";-import { useConfirmation } from "metabase/common/hooks/use-confirmation";import {ActionIcon,- Anchor,Box,Card,FixedSizeIcon,Group,- Loader,Menu,Stack,- Tooltip,} from "metabase/ui";import { getRelativeTime } from "metabase/utils/time-dayjs";import { getUserName } from "metabase/utils/user";-import {- useDeleteWorkspaceMutation,- useDeprovisionWorkspaceMutation,- useProvisionWorkspaceMutation,-} from "metabase-enterprise/api";import type { Workspace, WorkspaceDatabase } from "metabase-types/api";+import { trackWorkspaceConfigDownloaded } from "../../../analytics";import {- getDatabaseName,- getStatusMessage,- isDeprovisioned,- isDeprovisioning,- isProvisioned,- isProvisioning,+ getProvisioningFailureMessage,+ getWorkspaceDatabaseName,+ isUnprovisioned,} from "../../../utils";+import { DeleteWorkspaceModal } from "../DeleteWorkspaceModal";import { RenameWorkspaceModal } from "../RenameWorkspaceModal";-import { StatusDetailsModal } from "../StatusDetailsModal";++const CONFIG_FILENAME = "config.yml";export type WorkspaceItemProps = {workspace: Workspace;@@ -56,14 +47,7 @@ export function WorkspaceItem({ workspace }: WorkspaceItemProps) {{workspace.name}</Box><WorkspaceCreatorInfo workspace={workspace} />- {isProvisioned(workspace) ? (- <WorkspaceInstanceItem instanceUrl={workspace.instance_url} />- ) : (- <WorkspaceStatusItem workspace={workspace} />- )}- {workspace.target_branch != null && (- <WorkspaceBranchItem targetBranch={workspace.target_branch} />- )}+ {databases.some(isUnprovisioned) && <WorkspaceProvisioningWarning />}{databases.map((workspaceDatabase) => (<WorkspaceDatabaseItemkey={workspaceDatabase.database_id}@@ -94,96 +78,21 @@ function WorkspaceCreatorInfo({ workspace }: WorkspaceCreatorInfoProps) {);}-type WorkspaceStatusItemProps = {- workspace: Workspace;-};--function WorkspaceStatusItem({ workspace }: WorkspaceStatusItemProps) {- const showDetails = workspace.status_details != null;-- return (- <Group gap="xs" wrap="nowrap">- <WorkspaceStatusIcon workspace={workspace} />- <Box c="text-primary" lh="1rem">- {getStatusMessage(workspace.status)}- </Box>- {showDetails && <StatusDetailsButton workspace={workspace} />}- </Group>- );-}--function StatusDetailsButton({ workspace }: WorkspaceStatusItemProps) {- const [isDetailsOpen, { open: openDetails, close: closeDetails }] =- useDisclosure(false);-- return (- <>- <Tooltip label={t`See details`}>- <ActionIcon size="xs" aria-label={t`See details`} onClick={openDetails}>- <FixedSizeIcon name="info" aria-hidden />- </ActionIcon>- </Tooltip>- <StatusDetailsModal- workspace={workspace}- opened={isDetailsOpen}- onClose={closeDetails}- />- </>- );-}--function WorkspaceStatusIcon({ workspace }: WorkspaceStatusItemProps) {- if (isProvisioning(workspace) || isDeprovisioning(workspace)) {- return <Loader size="xs" />;- }- return <FixedSizeIcon name="warning" c="text-secondary" aria-hidden />;-}--type WorkspaceInstanceItemProps = {- instanceUrl: string | null;-};--function WorkspaceInstanceItem({ instanceUrl }: WorkspaceInstanceItemProps) {- return (- <Box c="text-secondary" lh="1rem">- <Group gap="xs" wrap="nowrap">- <FixedSizeIcon name="workspace" aria-hidden />- {instanceUrl != null ? (- <Anchor- href={instanceUrl}- target="_blank"- rel="noopener noreferrer"- lh="inherit"- >- {instanceUrl}- </Anchor>- ) : (- t`No workspace instance yet`- )}- </Group>- </Box>- );-}--type WorkspaceBranchItemProps = {- targetBranch: string;+type WorkspaceDatabaseItemProps = {+ workspaceDatabase: WorkspaceDatabase;};-function WorkspaceBranchItem({ targetBranch }: WorkspaceBranchItemProps) {+function WorkspaceProvisioningWarning() {return (<Box c="text-secondary" lh="1rem"><Group gap="xs" wrap="nowrap">- <FixedSizeIcon name="git_branch" aria-hidden />- {targetBranch}+ <FixedSizeIcon name="warning" aria-hidden />+ {getProvisioningFailureMessage()}</Group></Box>);}-type WorkspaceDatabaseItemProps = {- workspaceDatabase: WorkspaceDatabase;-};-function WorkspaceDatabaseItem({workspaceDatabase,}: WorkspaceDatabaseItemProps) {@@ -191,7 +100,7 @@ function WorkspaceDatabaseItem({<Box c="text-secondary" lh="1rem"><Group gap="xs" wrap="nowrap"><FixedSizeIcon name="database" aria-hidden />- {getDatabaseName(workspaceDatabase)}+ {getWorkspaceDatabaseName(workspaceDatabase)}</Group></Box>);@@ -204,69 +113,28 @@ type WorkspaceMenuProps = {function WorkspaceMenu({ workspace }: WorkspaceMenuProps) {const [isRenameOpen, { open: openRename, close: closeRename }] =useDisclosure(false);- const [provisionWorkspace, { isLoading: isProvisionLoading }] =- useProvisionWorkspaceMutation();- const [deprovisionWorkspace, { isLoading: isDeprovisionLoading }] =- useDeprovisionWorkspaceMutation();- const [deleteWorkspace, { isLoading: isDeleteLoading }] =- useDeleteWorkspaceMutation();- const { modalContent, show: showConfirmation } = useConfirmation();- const isInFlight = isProvisioning(workspace) || isDeprovisioning(workspace);-- const handleProvision = () => {- showConfirmation({- title: t`Provision this workspace?`,- message: t`This will set up temporary database users and schemas and a workspace instance.`,- confirmButtonText: t`Provision`,- confirmButtonProps: { color: "core-brand" },- onConfirm: () => provisionWorkspace(workspace.id),- });- };-- const handleDeprovision = () => {- showConfirmation({- title: t`Deprovision this workspace?`,- message: t`This will delete the workspace instance and the temporary database users and schemas that were created for this workspace.`,- confirmButtonText: t`Deprovision`,- onConfirm: () => deprovisionWorkspace(workspace.id),- });- };-- const handleDelete = () => {- showConfirmation({- title: t`Delete this workspace?`,- message: t`This will delete the workspace. This can't be undone.`,- confirmButtonText: t`Delete workspace`,- onConfirm: () => deleteWorkspace(workspace.id),- });- };+ const [isDeleteOpen, { open: openDelete, close: closeDelete }] =+ useDisclosure(false);return (<><Menu position="bottom-end"><Menu.Target>- <ActionIcon aria-label={t`Workspace actions`}>+ <ActionIcon aria-label={t`Workspace options`}><FixedSizeIcon name="ellipsis" aria-hidden /></ActionIcon></Menu.Target><Menu.Dropdown><Menu.Item- leftSection={<FixedSizeIcon name="play" aria-hidden />}- disabled={- isProvisioned(workspace) || isInFlight || isProvisionLoading- }- onClick={handleProvision}- >- {t`Provision`}- </Menu.Item>- <Menu.Item- leftSection={<FixedSizeIcon name="revert" aria-hidden />}- disabled={- isDeprovisioned(workspace) || isInFlight || isDeprovisionLoading+ component="a"+ href={`/api/ee/workspace-manager/${workspace.id}/config`}+ download={CONFIG_FILENAME}+ leftSection={<FixedSizeIcon name="download" aria-hidden />}+ onClick={() =>+ trackWorkspaceConfigDownloaded({ workspaceId: workspace.id })}- onClick={handleDeprovision}>- {t`Deprovision`}+ {t`Download ${CONFIG_FILENAME}`}</Menu.Item><Menu.ItemleftSection={<FixedSizeIcon name="pencil" aria-hidden />}@@ -276,8 +144,7 @@ function WorkspaceMenu({ workspace }: WorkspaceMenuProps) {</Menu.Item><Menu.ItemleftSection={<FixedSizeIcon name="trash" aria-hidden />}- disabled={!isDeprovisioned(workspace) || isDeleteLoading}- onClick={handleDelete}+ onClick={openDelete}>{t`Delete`}</Menu.Item>@@ -289,7 +156,13 @@ function WorkspaceMenu({ workspace }: WorkspaceMenuProps) {onRename={closeRename}onClose={closeRename}/>- {modalContent}+ {isDeleteOpen && (+ <DeleteWorkspaceModal+ workspaceId={workspace.id}+ onDelete={closeDelete}+ onClose={closeDelete}+ />+ )}</>);}diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceItem/WorkspaceItem.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceItem/WorkspaceItem.unit.spec.tsxindex d5e71692ef1..9aed518284b 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceItem/WorkspaceItem.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceItem/WorkspaceItem.unit.spec.tsx@@ -1,13 +1,7 @@import userEvent from "@testing-library/user-event";-import fetchMock from "fetch-mock";-import {- setupDeleteWorkspaceEndpoint,- setupDeprovisionWorkspaceEndpoint,- setupGetWorkspaceEndpoint,- setupProvisionWorkspaceEndpoint,-} from "__support__/server-mocks";-import { renderWithProviders, screen, waitFor } from "__support__/ui";+import { setupGetWorkspaceEndpoint } from "__support__/server-mocks";+import { renderWithProviders, screen } from "__support__/ui";import {createMockDatabase,createMockUserInfo,@@ -31,168 +25,6 @@ describe("WorkspaceItem", () => {expect(screen.getByText("My workspace")).toBeInTheDocument();});- it("provisions from the menu after confirmation", async () => {- const workspace = createMockWorkspace({- name: "My workspace",- status: "unprovisioned",- });- setupProvisionWorkspaceEndpoint(workspace);- setup({ workspace });-- await userEvent.click(- screen.getByRole("button", { name: "Workspace actions" }),- );- await userEvent.click(- await screen.findByRole("menuitem", { name: "Provision" }),- );-- expect(- await screen.findByRole("heading", { name: "Provision this workspace?" }),- ).toBeInTheDocument();- await userEvent.click(screen.getByRole("button", { name: "Provision" }));-- await waitFor(() => {- expect(- fetchMock.callHistory.called(- `path:/api/ee/workspace-manager/${workspace.id}/provision`,- { method: "POST" },- ),- ).toBe(true);- });- });-- it("deprovisions from the menu after confirmation", async () => {- const workspace = createMockWorkspace({- name: "My workspace",- status: "provisioned",- });- setupDeprovisionWorkspaceEndpoint(workspace);- setup({ workspace });-- await userEvent.click(- screen.getByRole("button", { name: "Workspace actions" }),- );- await userEvent.click(- await screen.findByRole("menuitem", { name: "Deprovision" }),- );-- expect(- await screen.findByRole("heading", {- name: "Deprovision this workspace?",- }),- ).toBeInTheDocument();- await userEvent.click(screen.getByRole("button", { name: "Deprovision" }));-- await waitFor(() => {- expect(- fetchMock.callHistory.called(- `path:/api/ee/workspace-manager/${workspace.id}/deprovision`,- { method: "POST" },- ),- ).toBe(true);- });- });-- it("disables Provision when provisioned and Deprovision when unprovisioned", async () => {- setup({- workspace: createMockWorkspace({- name: "My workspace",- status: "provisioned",- }),- });-- await userEvent.click(- screen.getByRole("button", { name: "Workspace actions" }),- );- expect(- await screen.findByRole("menuitem", { name: "Provision" }),- ).toBeDisabled();- expect(screen.getByRole("menuitem", { name: "Deprovision" })).toBeEnabled();- });-- it("disables Deprovision and enables Provision when unprovisioned", async () => {- setup({- workspace: createMockWorkspace({- name: "My workspace",- status: "unprovisioned",- }),- });-- await userEvent.click(- screen.getByRole("button", { name: "Workspace actions" }),- );- expect(- await screen.findByRole("menuitem", { name: "Deprovision" }),- ).toBeDisabled();- expect(screen.getByRole("menuitem", { name: "Provision" })).toBeEnabled();- });-- it("disables Provision and Deprovision while a run is in flight", async () => {- setup({- workspace: createMockWorkspace({- name: "My workspace",- status: "database-provisioning",- }),- });-- await userEvent.click(- screen.getByRole("button", { name: "Workspace actions" }),- );- expect(- await screen.findByRole("menuitem", { name: "Provision" }),- ).toBeDisabled();- expect(- screen.getByRole("menuitem", { name: "Deprovision" }),- ).toBeDisabled();- });-- it("shows status details in a modal via the See details button", async () => {- setup({- workspace: createMockWorkspace({- name: "My workspace",- status: "database-provisioning-failure",- status_details: "warehouse down",- }),- });-- expect(screen.getByText("Failed to set up databases")).toBeInTheDocument();- await userEvent.click(screen.getByRole("button", { name: "See details" }));-- const modal = await screen.findByRole("dialog");- expect(modal).toHaveTextContent("Failed to set up databases");- expect(modal).toHaveTextContent("warehouse down");- });-- it("does not offer status details when there are none", () => {- setup({- workspace: createMockWorkspace({- name: "My workspace",- status: "unprovisioned",- status_details: null,- }),- });-- expect(screen.getByText("Not provisioned")).toBeInTheDocument();- expect(- screen.queryByRole("button", { name: "See details" }),- ).not.toBeInTheDocument();- });-- it("shows the instance link instead of the status when provisioned", () => {- setup({- workspace: createMockWorkspace({- name: "My workspace",- status: "provisioned",- instance_url: "https://workspace.example.com",- }),- });-- expect(- screen.getByRole("link", { name: "https://workspace.example.com" }),- ).toBeInTheDocument();- expect(screen.queryByText("Provisioned")).not.toBeInTheDocument();- });-it("renders the creator info when the creator is hydrated", () => {setup({workspace: createMockWorkspace({@@ -215,29 +47,29 @@ describe("WorkspaceItem", () => {expect(screen.getByText(/^Created /)).toBeInTheDocument();});- it("renders the target branch when set", () => {- setup({- workspace: createMockWorkspace({- name: "My workspace",- target_branch: "feature-x",- }),- });-- expect(screen.getByText("feature-x")).toBeInTheDocument();- });-- it("does not render a branch block when the target branch is not set", () => {+ it("renders databases, falling back to the id when not hydrated", () => {setup({workspace: createMockWorkspace({+ id: 1,name: "My workspace",- target_branch: null,+ databases: [+ createMockWorkspaceDatabase({+ database_id: 10,+ database: createMockDatabase({ id: 10, name: "Postgres" }),+ }),+ createMockWorkspaceDatabase({+ database_id: 20,+ database: null,+ }),+ ],}),});- expect(screen.queryByText("feature-x")).not.toBeInTheDocument();+ expect(screen.getByText("Postgres")).toBeInTheDocument();+ expect(screen.getByText("Database 20")).toBeInTheDocument();});- it("renders databases, falling back to the id when not hydrated", () => {+ it("warns when a database is not provisioned", () => {setup({workspace: createMockWorkspace({id: 1,@@ -245,18 +77,17 @@ describe("WorkspaceItem", () => {databases: [createMockWorkspaceDatabase({database_id: 10,+ status: "unprovisioned",database: createMockDatabase({ id: 10, name: "Postgres" }),}),- createMockWorkspaceDatabase({- database_id: 20,- database: null,- }),],}),});+ expect(+ screen.getByText("Failed to provision the workspace."),+ ).toBeInTheDocument();expect(screen.getByText("Postgres")).toBeInTheDocument();- expect(screen.getByText("Database 20")).toBeInTheDocument();});it("does not warn when all databases are provisioned", () => {@@ -279,10 +110,26 @@ describe("WorkspaceItem", () => {).not.toBeInTheDocument();});+ it("offers a config download link in the menu", async () => {+ setup();+ await userEvent.click(+ screen.getByRole("button", { name: "Workspace options" }),+ );++ const downloadItem = await screen.findByRole("menuitem", {+ name: /Download config\.yml/,+ });+ expect(downloadItem).toHaveAttribute(+ "href",+ `/api/ee/workspace-manager/${WORKSPACE.id}/config`,+ );+ expect(downloadItem).toHaveAttribute("download", "config.yml");+ });+it("opens the rename modal from the menu", async () => {setup();await userEvent.click(- screen.getByRole("button", { name: "Workspace actions" }),+ screen.getByRole("button", { name: "Workspace options" }),);await userEvent.click(await screen.findByRole("menuitem", { name: "Rename" }),@@ -293,32 +140,10 @@ describe("WorkspaceItem", () => {).toBeInTheDocument();});- it("disables Delete unless the workspace is fully deprovisioned", async () => {- setup({- workspace: createMockWorkspace({- name: "My workspace",- status: "provisioned",- }),- });- await userEvent.click(- screen.getByRole("button", { name: "Workspace actions" }),- );-- expect(- await screen.findByRole("menuitem", { name: "Delete" }),- ).toBeDisabled();- });-- it("deletes an unprovisioned workspace from the menu after confirmation", async () => {- const workspace = createMockWorkspace({- name: "My workspace",- status: "unprovisioned",- });- setupDeleteWorkspaceEndpoint(workspace.id);- setup({ workspace });-+ it("opens the delete modal from the menu", async () => {+ setup();await userEvent.click(- screen.getByRole("button", { name: "Workspace actions" }),+ screen.getByRole("button", { name: "Workspace options" }),);await userEvent.click(await screen.findByRole("menuitem", { name: "Delete" }),@@ -327,17 +152,5 @@ describe("WorkspaceItem", () => {expect(await screen.findByRole("heading", { name: "Delete this workspace?" }),).toBeInTheDocument();- await userEvent.click(- screen.getByRole("button", { name: "Delete workspace" }),- );-- await waitFor(() => {- expect(- fetchMock.callHistory.called(- `path:/api/ee/workspace-manager/${workspace.id}`,- { method: "DELETE" },- ),- ).toBe(true);- });});});diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceListPage.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceListPage.tsxindex 22d74a748c3..8afb7eff734 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceListPage.tsx+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceListPage.tsx@@ -1,4 +1,3 @@-import { useEffect, useState } from "react";import { t } from "ttag";import { useListDatabasesQuery } from "metabase/api";@@ -10,32 +9,16 @@ import { Stack } from "metabase/ui";import { useListWorkspacesQuery } from "metabase-enterprise/api";import type { Database, Workspace } from "metabase-types/api";-import { POLLING_INTERVAL } from "../../constants";-import { isDeprovisioning, isProvisioning } from "../../utils";-import { NewWorkspaceButton } from "./NewWorkspaceButton";import { WorkspaceEmptyState } from "./WorkspaceEmptyState";import { WorkspaceItem } from "./WorkspaceItem";-function shouldPoll(workspaces: Workspace[] | undefined) {- return (workspaces ?? []).some(- (workspace) => isProvisioning(workspace) || isDeprovisioning(workspace),- );-}-export function WorkspaceListPage() {- const [isPolling, setIsPolling] = useState(false);const {data: workspaces,isLoading: isLoadingWorkspaces,error: workspacesError,- } = useListWorkspacesQuery(undefined, {- pollingInterval: isPolling ? POLLING_INTERVAL : undefined,- });-- useEffect(() => {- setIsPolling(shouldPoll(workspaces));- }, [workspaces]);+ } = useListWorkspacesQuery();const {data: databasesResponse,isLoading: isLoadingDatabases,diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceListPage.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceListPage.unit.spec.tsxindex 7cfc4024283..038f5e1e17c 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceListPage.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/pages/WorkspaceListPage/WorkspaceListPage.unit.spec.tsx@@ -6,7 +6,6 @@ import {setupListWorkspacesEndpoint,} from "__support__/server-mocks";import { renderWithProviders, screen } from "__support__/ui";-import { createMockSettingsState } from "metabase/redux/store/mocks";import { Route } from "metabase/router";import type { Workspace } from "metabase-types/api";import {@@ -29,9 +28,6 @@ function setup({ workspaces = [] as Workspace[] } = {}) {renderWithProviders(<Route path="*" element={<WorkspaceListPage />} />, {withRouter: true,- storeInitialState: {- settings: createMockSettingsState({ "remote-sync-enabled": true }),- },});}@@ -59,7 +55,7 @@ describe("WorkspaceListPage", () => {const item = await screen.findByRole("region", { name: "Existing" });expect(item).toBeInTheDocument();expect(- screen.getByRole("button", { name: "Workspace actions" }),+ screen.getByRole("button", { name: "Workspace options" }),).toBeInTheDocument();});});diff --git a/enterprise/frontend/src/metabase-enterprise/workspaces/utils.ts b/enterprise/frontend/src/metabase-enterprise/workspaces/utils.tsindex 9cfc4178541..b7a6c53d1c5 100644--- a/enterprise/frontend/src/metabase-enterprise/workspaces/utils.ts+++ b/enterprise/frontend/src/metabase-enterprise/workspaces/utils.ts@@ -1,99 +1,23 @@import { t } from "ttag";-import {- hasFeature,- hasWorkspacesEnabled,-} from "metabase/common/utils/database";-import type {- Database,- Workspace,- WorkspaceDatabase,- WorkspaceStatus,-} from "metabase-types/api";+import type { WorkspaceDatabase } from "metabase-types/api";-export function getDatabaseName(workspaceDatabase: WorkspaceDatabase) {+export function getWorkspaceDatabaseName(workspaceDatabase: WorkspaceDatabase) {return (workspaceDatabase.database?.name ??t`Database ${workspaceDatabase.database_id}`);}-export function getEligibleDatabases(databases: Database[]) {- return databases.filter(- (database) =>- hasFeature(database, "workspace") && hasWorkspacesEnabled(database),- );-}--export function isProvisioningFailed(workspace: Workspace) {- return (- workspace.status === "database-provisioning-failure" ||- workspace.status === "branch-provisioning-failure" ||- workspace.status === "instance-provisioning-failure"- );+export function isUnprovisioned(workspaceDatabase: WorkspaceDatabase) {+ return workspaceDatabase.status === "unprovisioned";}-export function isDeprovisioningFailed(workspace: Workspace) {- return (- workspace.status === "instance-deprovisioning-failure" ||- workspace.status === "branch-deprovisioning-failure" ||- workspace.status === "database-deprovisioning-failure"- );-}--export function isProvisioned(workspace: Workspace) {- return workspace.status === "provisioned";-}--export function isDeprovisioned(workspace: Workspace) {- return workspace.status === "unprovisioned";-}--export function isProvisioning(workspace: Workspace) {- return (- workspace.status === "database-provisioning" ||- workspace.status === "branch-provisioning" ||- workspace.status === "instance-provisioning"- );-}--export function isDeprovisioning(workspace: Workspace) {- return (- workspace.status === "instance-deprovisioning" ||- workspace.status === "branch-deprovisioning" ||- workspace.status === "database-deprovisioning"- );+export function isPending(workspaceDatabase: WorkspaceDatabase) {+ const { status } = workspaceDatabase;+ return status === "provisioning" || status === "deprovisioning";}-export function getStatusMessage(status: WorkspaceStatus): string {- switch (status) {- case "unprovisioned":- return t`Not provisioned`;- case "database-provisioning":- return t`Setting up databases…`;- case "database-provisioning-failure":- return t`Failed to set up databases`;- case "branch-provisioning":- return t`Creating the git branch…`;- case "branch-provisioning-failure":- return t`Failed to create the git branch`;- case "instance-provisioning":- return t`Setting up the instance…`;- case "instance-provisioning-failure":- return t`Failed to set up the instance`;- case "provisioned":- return t`Provisioned`;- case "instance-deprovisioning":- return t`Deprovisioning the instance…`;- case "instance-deprovisioning-failure":- return t`Failed to deprovision the instance`;- case "branch-deprovisioning":- return t`Deleting the git branch…`;- case "branch-deprovisioning-failure":- return t`Failed to delete the git branch`;- case "database-deprovisioning":- return t`Deprovisioning databases…`;- case "database-deprovisioning-failure":- return t`Failed to deprovision databases`;- }+export function getProvisioningFailureMessage() {+ return t`Failed to provision the workspace.`;}diff --git a/frontend/src/metabase-types/api/advanced-config.ts b/frontend/src/metabase-types/api/advanced-config.tsnew file mode 100644index 00000000000..ac4d17fa56c--- /dev/null+++ b/frontend/src/metabase-types/api/advanced-config.ts@@ -0,0 +1,26 @@+import type { CurrentWorkspaceDatabase } from "./workspace-instance";++export type AdvancedConfigDatabase = {+ name: string;+ engine: string;+ details: Record<string, unknown>;+};++export type AdvancedConfigWorkspace = {+ name: string;+ databases: Record<string, CurrentWorkspaceDatabase>;+};++export type AdvancedConfigContent = {+ databases?: AdvancedConfigDatabase[];+ workspace?: AdvancedConfigWorkspace;+};++export type AdvancedConfig = {+ version: number;+ config: AdvancedConfigContent;+};++export type ApplyAdvancedConfigRequest = {+ config: File;+};diff --git a/frontend/src/metabase-types/api/index.ts b/frontend/src/metabase-types/api/index.tsindex 974e2b57ed6..35bfd36b158 100644--- a/frontend/src/metabase-types/api/index.ts+++ b/frontend/src/metabase-types/api/index.ts@@ -1,6 +1,7 @@export * from "./actions";export * from "./activity";export * from "./admin";+export * from "./advanced-config";export * from "./advanced-permission";export * from "./ai-entity-analysis";export * from "./alert";diff --git a/frontend/src/metabase-types/api/mocks/workspace-instance.ts b/frontend/src/metabase-types/api/mocks/workspace-instance.tsindex c70ddbf2c52..a900d36b0ab 100644--- a/frontend/src/metabase-types/api/mocks/workspace-instance.ts+++ b/frontend/src/metabase-types/api/mocks/workspace-instance.ts@@ -20,6 +20,10 @@ export function createMockCurrentWorkspace(return {name: "Test workspace",databases: {},+ // Default to a non-writable workspace so tests that don't care about+ // lock state behave like a deployment-managed instance. Opt into the+ // editable case with createMockCurrentWorkspace({ can_write: true }).+ can_write: false,...opts,};}diff --git a/frontend/src/metabase-types/api/mocks/workspace.ts b/frontend/src/metabase-types/api/mocks/workspace.tsindex 32599f53495..137e2a1480b 100644--- a/frontend/src/metabase-types/api/mocks/workspace.ts+++ b/frontend/src/metabase-types/api/mocks/workspace.ts@@ -7,7 +7,6 @@ export function createMockWorkspaceDatabase(database_id: 1,input_schemas: [],status: "provisioned",- status_details: null,...opts,};}@@ -16,11 +15,6 @@ export function createMockWorkspace(opts?: Partial<Workspace>): Workspace {return {id: 1,name: "Test workspace",- target_branch: null,- status: "unprovisioned",- status_details: null,- instance_id: null,- instance_url: null,databases: [],created_at: "2026-01-01T00:00:00Z",creator_id: 1,diff --git a/frontend/src/metabase-types/api/workspace-instance.ts b/frontend/src/metabase-types/api/workspace-instance.tsindex 3f38bc4dbca..da6714e279e 100644--- a/frontend/src/metabase-types/api/workspace-instance.ts+++ b/frontend/src/metabase-types/api/workspace-instance.ts@@ -13,6 +13,7 @@ export type CurrentWorkspaceDatabase = {export type CurrentWorkspace = {name: string;databases: Record<DatabaseId, CurrentWorkspaceDatabase>;+ can_write: boolean;};export type GetCurrentWorkspaceResponse = {diff --git a/frontend/src/metabase-types/api/workspace-manager.ts b/frontend/src/metabase-types/api/workspace-manager.tsindex 0877d9d87fc..6e7f9d6bd2d 100644--- a/frontend/src/metabase-types/api/workspace-manager.ts+++ b/frontend/src/metabase-types/api/workspace-manager.ts@@ -6,32 +6,13 @@ export type WorkspaceId = number;export type WorkspaceDatabaseStatus =| "unprovisioned"| "provisioning"- | "provisioning-failure"| "provisioned"- | "deprovisioning"- | "deprovisioning-failure";--export type WorkspaceStatus =- | "unprovisioned"- | "database-provisioning"- | "database-provisioning-failure"- | "branch-provisioning"- | "branch-provisioning-failure"- | "instance-provisioning"- | "instance-provisioning-failure"- | "provisioned"- | "instance-deprovisioning"- | "instance-deprovisioning-failure"- | "branch-deprovisioning"- | "branch-deprovisioning-failure"- | "database-deprovisioning"- | "database-deprovisioning-failure";+ | "deprovisioning";export type WorkspaceDatabase = {database_id: DatabaseId;input_schemas: string[];status: WorkspaceDatabaseStatus;- status_details: string | null;database?: Database | null;};@@ -39,11 +20,6 @@ export type WorkspaceDatabase = {export type Workspace = {id: WorkspaceId;name: string;- target_branch: string | null;- status: WorkspaceStatus;- status_details: string | null;- instance_id: string | null;- instance_url: string | null;created_at: string;creator_id: UserId;@@ -53,7 +29,6 @@ export type Workspace = {export type CreateWorkspaceRequest = {name: string;- target_branch?: string;database_ids: DatabaseId[];};diff --git a/frontend/src/metabase/urls/admin.ts b/frontend/src/metabase/urls/admin.tsindex dc00f0cef85..0af27fc9a17 100644--- a/frontend/src/metabase/urls/admin.ts+++ b/frontend/src/metabase/urls/admin.ts@@ -147,10 +147,6 @@ export function uploadsSettings() {return "/admin/settings/uploads";}-export function remoteSyncSettings() {- return "/admin/settings/remote-sync";-}-export function adminLicense() {return "/admin/settings/license";}diff --git a/frontend/test/__support__/server-mocks/advanced-config.ts b/frontend/test/__support__/server-mocks/advanced-config.tsnew file mode 100644index 00000000000..8d7ebf7e788--- /dev/null+++ b/frontend/test/__support__/server-mocks/advanced-config.ts@@ -0,0 +1,5 @@+import fetchMock from "fetch-mock";++export function setupApplyAdvancedConfigEndpoint() {+ fetchMock.post("path:/api/ee/advanced-config", 204);+}diff --git a/frontend/test/__support__/server-mocks/index.ts b/frontend/test/__support__/server-mocks/index.tsindex b9b4c73d815..4877b0d5cc9 100644--- a/frontend/test/__support__/server-mocks/index.ts+++ b/frontend/test/__support__/server-mocks/index.ts@@ -1,5 +1,6 @@export * from "./action";export * from "./activity";+export * from "./advanced-config";export * from "./ai-entity-analysis";export * from "./alert";export * from "./api-key";diff --git a/frontend/test/__support__/server-mocks/workspace-instance.ts b/frontend/test/__support__/server-mocks/workspace-instance.tsindex 24bbc177571..69a00dae6da 100644--- a/frontend/test/__support__/server-mocks/workspace-instance.ts+++ b/frontend/test/__support__/server-mocks/workspace-instance.ts@@ -10,6 +10,10 @@ export function setupGetCurrentWorkspaceEndpoint(fetchMock.get(`${BASE_URL}/current`, { data: workspace });}+export function setupDeleteCurrentWorkspaceEndpoint() {+ fetchMock.delete(`${BASE_URL}/current`, 204);+}+export function setupListTableRemappingsEndpoint(remappings: TableRemapping[]) {fetchMock.get(`${BASE_URL}/table-remappings`, remappings);}diff --git a/frontend/test/__support__/server-mocks/workspace-manager.ts b/frontend/test/__support__/server-mocks/workspace-manager.tsindex b8890ca3c59..86b3c713937 100644--- a/frontend/test/__support__/server-mocks/workspace-manager.ts+++ b/frontend/test/__support__/server-mocks/workspace-manager.ts@@ -20,14 +20,6 @@ export function setupUpdateWorkspaceEndpoint(workspace: Workspace) {fetchMock.put(`${BASE_URL}/${workspace.id}`, workspace);}-export function setupProvisionWorkspaceEndpoint(workspace: Workspace) {- fetchMock.post(`${BASE_URL}/${workspace.id}/provision`, workspace);-}--export function setupDeprovisionWorkspaceEndpoint(workspace: Workspace) {- fetchMock.post(`${BASE_URL}/${workspace.id}/deprovision`, workspace);-}-export function setupDeleteWorkspaceEndpoint(workspaceId: WorkspaceId) {fetchMock.delete(`${BASE_URL}/${workspaceId}`, { status: 204 });}diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex d3b67ef4604..ce76fab3ab6 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -111,7 +111,6 @@eid-translationembeddingenterprise/api- enterprise/remote-syncenterprise/scimenterprise/serializationenterprise/ssodiff --git a/resources/migrations/064/20260721_drop_metadata_import_staging_tables.yaml b/resources/migrations/064/20260721_drop_metadata_import_staging_tables.yamlnew file mode 100644index 00000000000..6e784df79e4--- /dev/null+++ b/resources/migrations/064/20260721_drop_metadata_import_staging_tables.yaml@@ -0,0 +1,179 @@+## The metadata file importer (POST /api/ee/serialization/metadata/import) has+## been removed; drop its staging tables. The rollbacks recreate the tables+## exactly as 062/20260506_metadata_import_staging_tables.yaml defined them —+## they only ever held scratch data wiped on every import, so no data is lost+## in either direction.++databaseChangeLog:+ - changeSet:+ id: v64.2026-07-21T11:00:00+ author: ranquild+ comment: Drop the metabase_table_import staging table+ changes:+ - dropTable:+ tableName: metabase_table_import+ rollback:+ - createTable:+ tableName: metabase_table_import+ remarks: Staging table for the metadata file importer's tables phase. Drained outside the merge txn, merged into metabase_table inside it.+ columns:+ - column:+ name: source_id+ type: int+ constraints:+ nullable: false+ primaryKey: true+ primaryKeyName: pk_metabase_table_import+ - column:+ name: source_db_id+ type: int+ constraints:+ nullable: false+ - column:+ name: db_name+ type: varchar(254)+ constraints:+ nullable: false+ - column:+ name: schema+ type: varchar(254)+ constraints:+ nullable: true+ - column:+ name: name+ type: varchar(254)+ constraints:+ nullable: false+ - column:+ name: description+ type: ${text.type}+ constraints:+ nullable: true+ - column:+ name: display_name+ type: varchar(254)+ constraints:+ nullable: true+ - column:+ name: target_id+ type: int+ constraints:+ nullable: true+ - createIndex:+ tableName: metabase_table_import+ indexName: idx_mti_target_id+ columns:+ - column:+ name: target_id++ - changeSet:+ id: v64.2026-07-21T11:00:01+ author: ranquild+ comment: Drop the metabase_field_import staging table+ changes:+ - dropTable:+ tableName: metabase_field_import+ rollback:+ - createTable:+ tableName: metabase_field_import+ remarks: Staging table for the metadata file importer's fields phase. Drained outside the merge txn, depth-tagged, then merged into metabase_field level by level inside the txn.+ columns:+ - column:+ name: source_id+ type: int+ constraints:+ nullable: false+ primaryKey: true+ primaryKeyName: pk_metabase_field_import+ - column:+ name: source_table_id+ type: int+ constraints:+ nullable: false+ - column:+ name: source_parent_id+ type: int+ constraints:+ nullable: true+ - column:+ name: source_fk_target_id+ type: int+ constraints:+ nullable: true+ - column:+ name: name+ type: varchar(254)+ constraints:+ nullable: false+ - column:+ name: base_type+ type: varchar(254)+ constraints:+ nullable: false+ - column:+ name: database_type+ type: varchar(254)+ constraints:+ nullable: false+ - column:+ name: effective_type+ type: varchar(254)+ constraints:+ nullable: true+ - column:+ name: semantic_type+ type: varchar(254)+ constraints:+ nullable: true+ - column:+ name: coercion_strategy+ type: varchar(254)+ constraints:+ nullable: true+ - column:+ name: description+ type: ${text.type}+ constraints:+ nullable: true+ - column:+ name: nfc_path+ type: ${text.type}+ constraints:+ nullable: true+ - column:+ name: depth+ type: int+ constraints:+ nullable: true+ - column:+ name: target_id+ type: int+ constraints:+ nullable: true+ - column:+ name: target_table_id+ type: int+ constraints:+ nullable: true+ - column:+ name: target_parent_id+ type: int+ constraints:+ nullable: true+ - column:+ name: target_fk_target_id+ type: int+ constraints:+ nullable: true+ - createIndex:+ tableName: metabase_field_import+ indexName: idx_mfi_depth+ columns:+ - column:+ name: depth+ - createIndex:+ tableName: metabase_field_import+ indexName: idx_mfi_target_id+ columns:+ - column:+ name: target_iddiff --git a/resources/migrations/064/20260721_drop_workspace_async_columns.yaml b/resources/migrations/064/20260721_drop_workspace_async_columns.yamlnew file mode 100644index 00000000000..bbe9b345464--- /dev/null+++ b/resources/migrations/064/20260721_drop_workspace_async_columns.yaml@@ -0,0 +1,140 @@+## Reverse migrations for the "Async workspace provisioning" revert (#78172).+## The forward changesets from #78091 (20260718_workspace_status_columns.yaml,+## 20260720_workspace_target_branch.yaml) stay in the changelog so instances+## that already ran them keep consistent history; these changesets drop the+## added columns again. workspace_database.status keeps its widened+## varchar(254) — narrowing could truncate rows and the pre-revert code is+## fine with the wider type.++databaseChangeLog:+ - changeSet:+ id: v64.2026-07-21T10:00:00+ author: ranquild+ comment: Drop the unique index on workspace.target_branch+ changes:+ - dropIndex:+ tableName: workspace+ indexName: idx_workspace_target_branch+ rollback:+ - createIndex:+ tableName: workspace+ indexName: idx_workspace_target_branch+ unique: true+ columns:+ - column:+ name: target_branch++ - changeSet:+ id: v64.2026-07-21T10:00:01+ author: ranquild+ comment: Drop workspace.target_branch+ changes:+ - dropColumn:+ tableName: workspace+ columnName: target_branch+ rollback:+ - addColumn:+ tableName: workspace+ columns:+ - column:+ name: target_branch+ type: varchar(254)+ remarks: Git branch the workspace's child instance commits to via remote sync+ constraints:+ nullable: true++ - changeSet:+ id: v64.2026-07-21T10:00:02+ author: ranquild+ comment: Drop workspace.instance_url+ changes:+ - dropColumn:+ tableName: workspace+ columnName: instance_url+ rollback:+ - addColumn:+ tableName: workspace+ columns:+ - column:+ name: instance_url+ type: ${text.type}+ remarks: URL of the provisioned child Metabase instance+ constraints:+ nullable: true++ - changeSet:+ id: v64.2026-07-21T10:00:03+ author: ranquild+ comment: Drop workspace.instance_id+ changes:+ - dropColumn:+ tableName: workspace+ columnName: instance_id+ rollback:+ - addColumn:+ tableName: workspace+ columns:+ - column:+ name: instance_id+ type: varchar(254)+ remarks: External identifier of the provisioned child Metabase instance+ constraints:+ nullable: true++ - changeSet:+ id: v64.2026-07-21T10:00:04+ author: ranquild+ comment: Drop workspace.status_details+ changes:+ - dropColumn:+ tableName: workspace+ columnName: status_details+ rollback:+ - addColumn:+ tableName: workspace+ columns:+ - column:+ name: status_details+ type: ${text.type}+ remarks: Encrypted details for the current status, e.g. the provisioning failure message+ constraints:+ nullable: true++ - changeSet:+ id: v64.2026-07-21T10:00:05+ author: ranquild+ comment: Drop workspace.status+ changes:+ - dropColumn:+ tableName: workspace+ columnName: status+ rollback:+ - addColumn:+ tableName: workspace+ columns:+ - column:+ name: status+ type: varchar(254)+ defaultValue: unprovisioned+ remarks: Provisioning lifecycle state of the workspace+ constraints:+ nullable: false++ - changeSet:+ id: v64.2026-07-21T10:00:06+ author: ranquild+ comment: Drop workspace_database.status_details+ changes:+ - dropColumn:+ tableName: workspace_database+ columnName: status_details+ rollback:+ - addColumn:+ tableName: workspace_database+ columns:+ - column:+ name: status_details+ type: ${text.type}+ remarks: Encrypted details for the current status, e.g. the provisioning failure message+ constraints:+ nullable: truediff --git a/src/metabase/core/bootstrap.clj b/src/metabase/core/bootstrap.cljindex 5c9b719b19c..81998b79ae1 100644--- a/src/metabase/core/bootstrap.clj+++ b/src/metabase/core/bootstrap.clj@@ -34,7 +34,6 @@;;; etc.) and can run with minimal namespace loading for fast startup.;;;;;; Usage:-;;; java -jar metabase.jar --mode checker [checker-specific args...];;; java -jar metabase.jar --mode complexity-score [scorer-specific args...];;; ===========================================================================@@ -48,16 +47,13 @@The mode's -main function is responsible for calling System/exit."[mode args](let [startup (case mode- ;; schema checker was moved out to js. Perhaps this will get a long running server mode checker? Or- ;; perhaps just this one.- "checker" 'metabase-enterprise.checker.cli/entrypoint"complexity-score" 'metabase-enterprise.data-complexity-score.cli/entrypointnil)](if startup((requiring-resolve startup) args)(do (binding [*out* *err*](output! (str "Unknown mode: " mode))- (output! "Available modes: checker, complexity-score"))+ (output! "Available modes: complexity-score"))(System/exit 1)))))(defn -maindiff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex 1ae9d9582c0..cf7b82c2f6f 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -2048,9 +2048,9 @@Callers are expected to bind [[metabase.driver.connection/with-admin-connection]]around this call so connections resolve against the database's `:admin-details`- overlay when configured. The default `database-provisioner` in- `metabase-enterprise.workspaces.provisioning.database` does this; do not rely- on default connection details from inside impls."+ overlay when configured. The default `dispatching-provisioner` in+ `metabase-enterprise.workspaces.provisioning` does this; do not rely on+ default connection details from inside impls."{:added "0.59.0" :arglists '([driver database workspace])}dispatch-on-initialized-driver:hierarchy #'hierarchy)diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.cljindex 8b2ffd55587..f483a45bbd4 100644--- a/src/metabase/models/interface.clj+++ b/src/metabase/models/interface.clj@@ -327,11 +327,6 @@{:in encrypted-json-in:out cached-encrypted-json-out})-(def transform-encrypted-text- "Transform for encrypted plain-text columns."- {:in encryption/maybe-encrypt- :out encryption/maybe-decrypt})-;;; TODO (Cam 10/27/25) -- this stuff should be moved into a different module instead of the general models interface,;;; either `queries` or a new module along with [[metabase.models.visualization-settings]].(mr/def ::viz-settings-refdiff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex c99b5a8941f..cd33bd864b7 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -671,36 +671,6 @@(defmethod serialization-dependencies :default [_ _]nil)-;;; # Metadata Export--(defmulti metadata-query- "Returns a reducible row stream for `model` for the metadata export endpoint.- `opts` carries caller context (e.g. `:user-info`, section flags, id allow-lists).- Implementations return rows with raw numeric ids — joins for portable identifiers- are intentionally avoided so the queries stay simple."- {:arglists '([model opts])}- (fn [model _opts] model))--(defmulti metadata-query-filter- "Returns a HoneySQL where predicate that restricts rows of `model` qualified by- table `alias`. Used to compose `WHERE` and `JOIN ... ON` clauses across model- hierarchies (e.g. the field query reuses the database/table filters)."- {:arglists '([model alias opts])}- (fn [model _alias _opts] model))--(defmulti metadata-query-format- "Reshapes a raw query row produced by [[metadata-query]] into the JSON shape emitted- by the metadata export endpoint. The default drops nil-valued keys via- [[metabase.util/remove-nils]]; overrides do JSON decoding or column-derived- adjustments and rely on the same nil-pruning to keep optional keys out of the- response."- {:arglists '([model row])}- (fn [model _row] model))--(defmethod metadata-query-format :default- [_model row]- (u/remove-nils row))-;;; # Import Process;;; Deserialization is split into two stages, mirroring serialization. They are called *ingestion* and *loading*.;;; Ingestion turns whatever serialized form was produced by storage (eg. a tree of YAML files) into Clojure maps withdiff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.cljindex 69f2f7739e3..c4b9379aee6 100644--- a/src/metabase/warehouse_schema/models/field.clj+++ b/src/metabase/warehouse_schema/models/field.clj@@ -14,7 +14,6 @@[metabase.remote-sync.core :as remote-sync][metabase.util :as u][metabase.util.i18n :refer [tru]]- [metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]@@ -496,42 +495,3 @@(conj (serdes/storage-path-prefixes path){:label "fields"}{:label field-name :key field-name})))--(defmethod serdes/metadata-query :model/Field- [model opts]- (t2/reducible-query- {:select [[:f.id :id]- [:f.table_id :table_id]- [:f.name :name]- [:f.parent_id :parent_id]- [:f.fk_target_field_id :fk_target_field_id]- [:f.description :description]- [:f.base_type :base_type]- [:f.database_type :database_type]- [:f.effective_type :effective_type]- [:f.semantic_type :semantic_type]- [:f.coercion_strategy :coercion_strategy]- [:f.nfc_path :nfc_path]]- :from [[(t2/table-name model) :f]]- :join [[(t2/table-name :model/Table) :t] [:= :f.table_id :t.id]- [(t2/table-name :model/Database) :db] [:= :t.db_id :db.id]]- :where [:and- (serdes/metadata-query-filter :model/Database :db opts)- (serdes/metadata-query-filter :model/Table :t opts)- (serdes/metadata-query-filter model :f opts)]}))--(defmethod serdes/metadata-query-filter :model/Field- [_model alias {:keys [field-ids]}]- (cond-> [:and- [:= (u/qualified-key alias :active) true]- [:<> (u/qualified-key alias :visibility_type) "sensitive"]]- (seq field-ids) (conj [:in (u/qualified-key alias :id) field-ids])))--(defmethod serdes/metadata-query-format :model/Field- [_model {:keys [base_type effective_type nfc_path] :as row}]- (-> row- (assoc :effective_type (when (not= base_type effective_type) effective_type))- (assoc :nfc_path (cond-> nfc_path- (string? nfc_path) json/decode- nfc_path seq))- u/remove-nils))diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex 3079c618929..dea1aa1da01 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -644,36 +644,6 @@(conj (serdes/storage-path-prefixes (serdes/path table)){:label (:name table) :key (:name table)}))-(defmethod serdes/metadata-query :model/Table- [model opts]- (t2/reducible-query- {:select [[:t.id :id]- [:t.db_id :db_id]- [:t.name :name]- [:t.schema :schema]- [:t.description :description]]- :from [[(t2/table-name model) :t]]- :join [[(t2/table-name :model/Database) :db] [:= :t.db_id :db.id]]- :where [:and- (serdes/metadata-query-filter :model/Database :db opts)- (serdes/metadata-query-filter model :t opts)]}))--(defmethod serdes/metadata-query-filter :model/Table- [_model alias {:keys [user-info table-ids schema-ids]}]- (let [perm-mapping {:perms/view-data :unrestricted- :perms/create-queries :query-builder}]- (cond-> [:and- [:= (u/qualified-key alias :active) true]- [:= (u/qualified-key alias :visibility_type) nil]- [:in (u/qualified-key alias :id)- (perms/visible-table-filter-select :id user-info perm-mapping)]]- (seq schema-ids) (conj (into [:or]- (for [[db-id schemas] schema-ids]- [:and- [:= (u/qualified-key alias :db_id) db-id]- [:in (u/qualified-key alias :schema) schemas]])))- (seq table-ids) (conj [:in (u/qualified-key alias :id) table-ids]))))-;;;; ------------------------------------------------- Search ----------------------------------------------------------(search.spec/define-spec "table"diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 2c8d2170cae..128825dd8d2 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -735,25 +735,6 @@(:admin_details ingested) (update :admin_details driver/sanitize-db-details))maybe-local))-(def ^:private metadata-export-perms- {:perms/view-data :unrestricted- :perms/create-queries :query-builder})--(defmethod serdes/metadata-query :model/Database- [model opts]- (t2/reducible-query {:select [:id :name :engine]- :from [[(t2/table-name model) :db]]- :where (serdes/metadata-query-filter model :db opts)}))--(defmethod serdes/metadata-query-filter :model/Database- [_model alias {:keys [user-info database-ids]}]- (cond-> [:and- [:= (u/qualified-key alias :is_audit) false]- [:= (u/qualified-key alias :router_database_id) nil]- [:in (u/qualified-key alias :id)- (perms/visible-database-filter-select user-info metadata-export-perms)]]- (seq database-ids) (conj [:in (u/qualified-key alias :id) database-ids])))-(def ^{:arglists '([table-id])} table-id->database-id"Retrieve the `Database` ID for the given table-id."(mdb/memoize-for-application-dbdiff --git a/src/metabase/workspaces/core.clj b/src/metabase/workspaces/core.cljindex 17028c7f68c..ba30d97aee6 100644--- a/src/metabase/workspaces/core.clj+++ b/src/metabase/workspaces/core.clj@@ -35,7 +35,7 @@## OSS vs EEThe OSS jar has no workspace mode (`workspace-mode?` returns false), so- the gates are no-ops. The EE impl in `metabase-enterprise.workspaces.instance`+ the gates are no-ops. The EE impl in `metabase-enterprise.workspaces.core`reads the `instance-workspace` setting."(:require[metabase.premium-features.core :refer [defenterprise]]@@ -81,12 +81,12 @@was loaded from `config.yml` at boot. Instance-level state set once at boot.For the per-database check, use the EE function- `metabase-enterprise.workspaces.instance/db-workspace-namespace` (returns the+ `metabase-enterprise.workspaces.core/db-workspace-namespace` (returns theworkspace-isolated output namespace map if this instance is in workspace modeAND the workspace's `:databases` map includes this database).The OSS fallback returns false."- metabase-enterprise.workspaces.instance+ metabase-enterprise.workspaces.core[]false)