The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Make data apps regular git sync entities (#83455)
Alexander Polyankina14a43b8f38c
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 46a8384e461..baa037bc04a 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1723,6 +1723,7 @@:model/DashboardCard:model/DashboardCardSeries:model/DashboardTab+ :model/DataApp:model/Database:model/Dimension:model/Document@@ -3736,10 +3737,11 @@enterprise/data-apps{:team "Embedding"- :api #{metabase-enterprise.data-apps.api metabase-enterprise.data-apps.init metabase-enterprise.data-apps.sync}+ :api #{metabase-enterprise.data-apps.api metabase-enterprise.data-apps.init}:uses #{apiapi-scopecollections+ eventsliblib.belib.schema@@ -3757,7 +3759,8 @@:model/Permissions:model/PermissionsGroup:model/Table- :model/User}}+ :model/User}+ :model-exports #{:model/DataApp}}enterprise/data-studio{:team "UX West"@@ -4193,7 +4196,6 @@task.historyutilwarehouse-schema-overlay- enterprise/data-appsenterprise/serializationenterprise/transforms.python}:model-exports #{:model/RemoteSyncTask}@@ -4201,6 +4203,7 @@:model/Card:model/Collection:model/Dashboard+ :model/DataApp:model/Document:model/Field:model/FieldUserSettingsdiff --git a/e2e/embedding-sdk-host-apps/vite-6-data-app-host-app/data_app.yaml b/e2e/embedding-sdk-host-apps/vite-6-data-app-host-app/data_app.yamlindex 112f1b2c983..9a2498dc118 100644--- a/e2e/embedding-sdk-host-apps/vite-6-data-app-host-app/data_app.yaml+++ b/e2e/embedding-sdk-host-apps/vite-6-data-app-host-app/data_app.yaml@@ -1,5 +1,11 @@-name: Vite 6 Data Appversion: 1+name: Vite 6 Data App+slug: vite-6-data-app-host-apppath: ./dist/index.jsallowed_hosts:- https://allowed.data-app.test+entity_id: qxpaPkU_WRE2ZQu0cmpqD+serdes/meta:+- model: DataApp+ id: qxpaPkU_WRE2ZQu0cmpqD+ label: vite-6-data-app-host-appdiff --git a/e2e/support/assets/data-apps/custom-error-component/data_app.yaml b/e2e/support/assets/data-apps/custom-error-component/data_app.yamlindex 1a604fccf95..6e534facce0 100644--- a/e2e/support/assets/data-apps/custom-error-component/data_app.yaml+++ b/e2e/support/assets/data-apps/custom-error-component/data_app.yaml@@ -1,3 +1,9 @@-name: Custom Error Componentversion: 1+name: Custom Error Component+slug: custom-error-componentpath: ./dist/index.js+entity_id: uWb09GFf1aqUzlNsbhPyF+serdes/meta:+- model: DataApp+ id: uWb09GFf1aqUzlNsbhPyF+ label: custom-error-componentdiff --git a/e2e/support/assets/data-apps/isolation/data_app.yaml b/e2e/support/assets/data-apps/isolation/data_app.yamlindex 96961791805..3ff1fd0cd67 100644--- a/e2e/support/assets/data-apps/isolation/data_app.yaml+++ b/e2e/support/assets/data-apps/isolation/data_app.yaml@@ -1,3 +1,9 @@-name: Isolationversion: 1+name: Isolation+slug: isolationpath: ./dist/index.js+entity_id: nu2o-JSPcsIfu3FM0ek24+serdes/meta:+- model: DataApp+ id: nu2o-JSPcsIfu3FM0ek24+ label: isolationdiff --git a/e2e/support/assets/data-apps/kitchen-sink/data_app.yaml b/e2e/support/assets/data-apps/kitchen-sink/data_app.yamlindex 13a6b1d1769..b63b7271490 100644--- a/e2e/support/assets/data-apps/kitchen-sink/data_app.yaml+++ b/e2e/support/assets/data-apps/kitchen-sink/data_app.yaml@@ -1,3 +1,9 @@-name: Kitchen Sinkversion: 1+name: Kitchen Sink+slug: kitchen-sinkpath: ./dist/index.js+entity_id: JyvrOkPvwWQAQCwUEd1bb+serdes/meta:+- model: DataApp+ id: JyvrOkPvwWQAQCwUEd1bb+ label: kitchen-sinkdiff --git a/e2e/support/assets/data-apps/synced-actions-app/data_app.yaml b/e2e/support/assets/data-apps/synced-actions-app/data_app.yamlindex c0811b7f749..c8bf7f8c6da 100644--- a/e2e/support/assets/data-apps/synced-actions-app/data_app.yaml+++ b/e2e/support/assets/data-apps/synced-actions-app/data_app.yaml@@ -1,3 +1,9 @@-name: Synced Actions Appversion: 1+name: Synced Actions App+slug: synced-actions-apppath: ./dist/index.js+entity_id: nPu5r1EndcM_D8TQfLk9c+serdes/meta:+- model: DataApp+ id: nPu5r1EndcM_D8TQfLk9c+ label: synced-actions-appdiff --git a/e2e/support/assets/data-apps/synced-app/data_app.yaml b/e2e/support/assets/data-apps/synced-app/data_app.yamlindex 3ff729ed314..3e282a17794 100644--- a/e2e/support/assets/data-apps/synced-app/data_app.yaml+++ b/e2e/support/assets/data-apps/synced-app/data_app.yaml@@ -1,3 +1,9 @@-name: Synced Appversion: 1+name: Synced App+slug: synced-apppath: ./dist/index.js+entity_id: 4vxutiAe703DZgttXwORO+serdes/meta:+- model: DataApp+ id: 4vxutiAe703DZgttXwORO+ label: synced-appdiff --git a/e2e/support/assets/example_synced_data_apps/data_apps/bad-config/data_app.yaml b/e2e/support/assets/example_synced_data_apps/data_apps/bad-config/data_app.yamldeleted file mode 100644index 89a1e6b0e9d..00000000000--- a/e2e/support/assets/example_synced_data_apps/data_apps/bad-config/data_app.yaml+++ /dev/null@@ -1 +0,0 @@-name: [unterminateddiff --git a/e2e/support/assets/example_synced_data_apps/data_apps/bad-config/index.js b/e2e/support/assets/example_synced_data_apps/data_apps/bad-config/index.jsdeleted file mode 100644index 2d569610c95..00000000000--- a/e2e/support/assets/example_synced_data_apps/data_apps/bad-config/index.js+++ /dev/null@@ -1,2 +0,0 @@-// Present, but unreachable: its data_app.yaml can't be parsed, so the app is skipped.-export default () => ({ component: null });diff --git a/e2e/support/assets/example_synced_data_apps/data_apps/broken-bundle/data_app.yaml b/e2e/support/assets/example_synced_data_apps/data_apps/broken-bundle/data_app.yamldeleted file mode 100644index d95ac47d8d7..00000000000--- a/e2e/support/assets/example_synced_data_apps/data_apps/broken-bundle/data_app.yaml+++ /dev/null@@ -1,4 +0,0 @@-name: Broken Bundle-version: 1-# Points at a file that isn't in the repo — a valid config, an un-loadable app.-path: ./missing.jsdiff --git a/e2e/support/assets/example_synced_data_apps/data_apps/good/data_app.yaml b/e2e/support/assets/example_synced_data_apps/data_apps/good/data_app.yamlindex 8d241b3ad25..712c4c8a6c4 100644--- a/e2e/support/assets/example_synced_data_apps/data_apps/good/data_app.yaml+++ b/e2e/support/assets/example_synced_data_apps/data_apps/good/data_app.yaml@@ -1,4 +1,10 @@+version: 1name: Good App+slug: gooddescription: A well-formed app that syncs cleanly-version: 1path: ./index.js+entity_id: Ioxf30LzIQCGwbCNtaG62+serdes/meta:+- model: DataApp+ id: Ioxf30LzIQCGwbCNtaG62+ label: gooddiff --git a/e2e/support/assets/example_synced_data_apps/data_apps/no-config/index.js b/e2e/support/assets/example_synced_data_apps/data_apps/no-config/index.jsdeleted file mode 100644index 9c1c2183649..00000000000--- a/e2e/support/assets/example_synced_data_apps/data_apps/no-config/index.js+++ /dev/null@@ -1,3 +0,0 @@-// A bundle with no data_app.yaml beside it — nothing declares this directory an-// app, so sync doesn't discover it.-export default () => ({ component: null });diff --git a/e2e/support/assets/example_synced_data_apps/data_apps/second-app/data_app.yaml b/e2e/support/assets/example_synced_data_apps/data_apps/second-app/data_app.yamlnew file mode 100644index 00000000000..57d564db974--- /dev/null+++ b/e2e/support/assets/example_synced_data_apps/data_apps/second-app/data_app.yaml@@ -0,0 +1,9 @@+version: 1+name: Second App+slug: second-app+path: ./index.js+entity_id: A-0Jd9Jt80EIbt1hOavWS+serdes/meta:+- model: DataApp+ id: A-0Jd9Jt80EIbt1hOavWS+ label: second-appdiff --git a/e2e/support/assets/example_synced_data_apps/data_apps/second-app/index.js b/e2e/support/assets/example_synced_data_apps/data_apps/second-app/index.jsnew file mode 100644index 00000000000..e68ee3ca4c4--- /dev/null+++ b/e2e/support/assets/example_synced_data_apps/data_apps/second-app/index.js@@ -0,0 +1,3 @@+// A placeholder bundle — this fixture exercises sync/materialization + the admin+// listing + the bundle endpoint, not rendering, so the bytes are served opaquely.+export default () => ({ component: null });diff --git a/e2e/support/helpers/e2e-data-app-helpers.ts b/e2e/support/helpers/e2e-data-app-helpers.tsindex f212c0f8819..8bcb1933a59 100644--- a/e2e/support/helpers/e2e-data-app-helpers.ts+++ b/e2e/support/helpers/e2e-data-app-helpers.ts@@ -23,21 +23,20 @@ export const visitDataAppRoute = (route: string) => {export const fakeDataApp = (overrides: Partial<DataApp> = {}): DataApp => ({id: 1,+ entity_id: "e2eFakeDataAppEntityI",name: DATA_APP_NAME,display_name: DATA_APP_DISPLAY_NAME,description: null,version: 1,outdated: false,- bundle_path: `data_apps/${DATA_APP_NAME}/dist/index.js`,+ bundle_path: "dist/index.js",enabled: true,+ draft: false,resource_collection_id: null,permission_group_id: null,table_ids: [],allowed_hosts: [],bundle_hash: "e2e-bundle-hash",- last_synced_sha: "e2e0000",- last_synced_at: "2024-01-01T00:00:00Z",- sync_error: null,created_at: "2024-01-01T00:00:00Z",updated_at: "2024-01-01T00:00:00Z",...overrides,@@ -358,7 +357,7 @@ export function createDataAppApiKey() {/*** A second app beside the host app, for cases that need two of them. It reuses* the host app's `node_modules`, so `defineQuery` still resolves through the- * published SDK, and its directory name becomes the app's slug.+ * published SDK, and its manifest declares `slug` as its slug.*/export function createSecondDataApp(slug: string) {cy.task("scaffoldDataApp", { appName: slug, sdkFrom: dataAppHostAppRoot() });diff --git a/e2e/support/helpers/e2e-data-app-tasks.ts b/e2e/support/helpers/e2e-data-app-tasks.tsindex 3e6ad76bec4..da800cc0f66 100644--- a/e2e/support/helpers/e2e-data-app-tasks.ts+++ b/e2e/support/helpers/e2e-data-app-tasks.ts@@ -120,7 +120,7 @@ export async function scaffoldDataApp({);fs.writeFileSync(path.join(root, "data_app.yaml"),- `name: ${appName}\npath: ./dist/index.js\n`,+ `name: ${appName}\nslug: ${appName}\npath: ./dist/index.js\n`,);return root;diff --git a/e2e/test-host-app/data-apps/data-app-dev-diagnostics.cy.spec.ts b/e2e/test-host-app/data-apps/data-app-dev-diagnostics.cy.spec.tsindex f411c7719f2..ae16093dcf4 100644--- a/e2e/test-host-app/data-apps/data-app-dev-diagnostics.cy.spec.ts+++ b/e2e/test-host-app/data-apps/data-app-dev-diagnostics.cy.spec.ts@@ -262,10 +262,12 @@ describe("Embedding SDK: data-app dev diagnostics", () => {.findByText(/allowed_hosts changed since the dev server started/).should("not.exist");- // `allowed_hosts` is the manifest's last key, so appending stays valid YAML.cy.writeFile(DATA_APP_DEV_MANIFEST_PATH,- `${originalManifest} - https://added.example\n`,+ originalManifest.replace(+ " - https://allowed.data-app.test\n",+ " - https://allowed.data-app.test\n - https://added.example\n",+ ),);// No reload and no dev-server restart: the watcher re-validates and thediff --git a/e2e/test-host-app/data-apps/sync-resources-actions.cy.spec.ts b/e2e/test-host-app/data-apps/sync-resources-actions.cy.spec.tsindex a99c4df0500..314808d2bab 100644--- a/e2e/test-host-app/data-apps/sync-resources-actions.cy.spec.ts+++ b/e2e/test-host-app/data-apps/sync-resources-actions.cy.spec.ts@@ -17,7 +17,7 @@ const { H } = cy;const TEST_TABLE = "scoreboard_actions";const MODEL_NAME = "Scoreboard model";-/** `syncResources` takes the app's slug from its directory name. */+/** The `slug` the app's `data_app.yaml` declares. */const APP_SLUG = "vite-6-data-app-host-app";const APP_ROOT = () => dataAppHostAppRoot();diff --git a/e2e/test-host-app/data-apps/sync-resources.cy.spec.ts b/e2e/test-host-app/data-apps/sync-resources.cy.spec.tsindex c7cf5639ff8..18179e1bf9b 100644--- a/e2e/test-host-app/data-apps/sync-resources.cy.spec.ts+++ b/e2e/test-host-app/data-apps/sync-resources.cy.spec.ts@@ -18,7 +18,7 @@ import type { Card, DataApp } from "metabase-types/api";const { H } = cy;const { ORDERS_ID } = SAMPLE_DATABASE;-/** `syncResources` takes the app's slug from its directory name. */+/** The `slug` the app's `data_app.yaml` declares. */const APP_SLUG = "vite-6-data-app-host-app";/** The fields these specs read off a card the app collection holds. */@@ -29,10 +29,17 @@ const LOCKFILE = () => `${APP_ROOT()}/resources_metadata.json`;const QUERIES_FILE = () => `${APP_ROOT()}/queries/orders.query.ts`;const MANIFEST_FILE = () => `${APP_ROOT()}/data_app.yaml`;-const AUTHORED_MANIFEST = `name: Vite 6 Data App+const AUTHORED_MANIFEST = `version: 1+name: Vite 6 Data App+slug: vite-6-data-app-host-apppath: ./dist/index.jsallowed_hosts:- https://allowed.data-app.test+entity_id: qxpaPkU_WRE2ZQu0cmpqD+serdes/meta:+- model: DataApp+ id: qxpaPkU_WRE2ZQu0cmpqD+ label: vite-6-data-app-host-app`;/**diff --git a/e2e/test/scenarios/data-apps/sync-resources-production-actions.cy.spec.ts b/e2e/test/scenarios/data-apps/sync-resources-production-actions.cy.spec.tsindex d538c8c5ee2..3d57e6bd719 100644--- a/e2e/test/scenarios/data-apps/sync-resources-production-actions.cy.spec.ts+++ b/e2e/test/scenarios/data-apps/sync-resources-production-actions.cy.spec.ts@@ -23,7 +23,18 @@ const MANIFEST_FILE = () => `${APP_ROOT()}/data_app.yaml`;const LOCKFILE = () => `${APP_ROOT()}/resources_metadata.json`;/** The manifest as source control holds it. */-const AUTHORED_MANIFEST = `name: ${APP_DISPLAY_NAME}\npath: ./dist/index.js\n`;+const AUTHORED_MANIFEST = [+ "version: 1",+ `name: ${APP_DISPLAY_NAME}`,+ `slug: ${APP_SLUG}`,+ "path: ./dist/index.js",+ "entity_id: nPu5r1EndcM_D8TQfLk9c",+ "serdes/meta:",+ "- model: DataApp",+ " id: nPu5r1EndcM_D8TQfLk9c",+ " label: synced-actions-app",+ "",+].join("\n");/** The declaration as source control holds it, with no generated ID yet. */const declaration = (sourceActionId: number) =>diff --git a/e2e/test/scenarios/data-apps/sync-resources-production.cy.spec.ts b/e2e/test/scenarios/data-apps/sync-resources-production.cy.spec.tsindex fd1d0decb94..b2d292e1386 100644--- a/e2e/test/scenarios/data-apps/sync-resources-production.cy.spec.ts+++ b/e2e/test/scenarios/data-apps/sync-resources-production.cy.spec.ts@@ -22,7 +22,18 @@ const MANIFEST_FILE = () => `${APP_ROOT()}/data_app.yaml`;const LOCKFILE = () => `${APP_ROOT()}/resources_metadata.json`;/** The manifest as source control holds it. */-const AUTHORED_MANIFEST = `name: ${APP_DISPLAY_NAME}\npath: ./dist/index.js\n`;+const AUTHORED_MANIFEST = [+ "version: 1",+ `name: ${APP_DISPLAY_NAME}`,+ `slug: ${APP_SLUG}`,+ "path: ./dist/index.js",+ "entity_id: 4vxutiAe703DZgttXwORO",+ "serdes/meta:",+ "- model: DataApp",+ " id: 4vxutiAe703DZgttXwORO",+ " label: synced-app",+ "",+].join("\n");/** The declaration as source control holds it, with no generated ID yet. */const AUTHORED_DECLARATION = [diff --git a/e2e/test/scenarios/data-apps/sync.cy.spec.ts b/e2e/test/scenarios/data-apps/sync.cy.spec.tsindex bed4ffb67ee..325ba2a13b1 100644--- a/e2e/test/scenarios/data-apps/sync.cy.spec.ts+++ b/e2e/test/scenarios/data-apps/sync.cy.spec.ts@@ -1,17 +1,8 @@const { H } = cy;/**- * Drives a real remote-sync pull of a repo whose `data_apps/` covers every- * materialization outcome, and asserts each is handled the way the backend- * intends (see `data-apps.sync` / `data-apps.config`):- *- * good/ valid config + bundle -> materialized, served- * broken-bundle/ valid config, missing bundle -> row with "Sync failed", not served- * bad-config/ malformed data_app.yaml -> skipped, no row- * no-config/ a bundle but no data_app.yaml -> not discovered, no row- *- * A bad app never blocks a good one, and neither a bad config nor a missing- * bundle removes an app that isn't theirs.+ * Drives a real remote-sync pull of a repo whose `data_apps/` holds two apps, each a `data_app.yaml` with its bundle+ * next to it, and asserts that pulls materialize and remove them.*/describe("scenarios > data apps > repo sync", () => {beforeEach(() => {@@ -21,18 +12,15 @@ describe("scenarios > data apps > repo sync", () => {H.setupGitSync();});- it("materializes each app per its config/bundle, isolating the broken ones", () => {+ it("materializes each app with its bundle", () => {H.copySyncedCollectionFixture();H.copySyncedDataAppsFixture();- H.commitToRepo("Add data apps with mixed config/bundle states");+ H.commitToRepo("Add data apps");H.configureGitAndPullChanges("read-write");cy.visit("/admin/settings/apps");cy.findByTestId("admin-layout-content").within(() => {- // The good app is materialized and shown as synced. Scope to its own row so- // its status can't be satisfied by another app's — a "Sync failed" leaking- // onto the good app (or "Synced" onto the broken one) must fail the test.cy.findByTestId("data-app-list-item-good").scrollIntoView().within(() => {@@ -40,97 +28,37 @@ describe("scenarios > data apps > repo sync", () => {cy.findByText("A well-formed app that syncs cleanly").should("be.visible",);- cy.findByText(/^Synced/).should("be.visible");- });-- // The app whose bundle is missing still appears — with its failure, not hidden.- // Its name is plain text, not a link: a sync-failed app can't be opened.- cy.findByTestId("data-app-list-item-broken-bundle")- .scrollIntoView()- .within(() => {- cy.findByText("Broken Bundle").should("be.visible");- cy.findByRole("link", { name: "Broken Bundle" }).should("not.exist");- cy.findByText("Sync failed").should("be.visible");});-- // The malformed config and the config-less directory produced no app at all.- cy.findByText("/apps/bad-config").should("not.exist");- cy.findByText("/apps/no-config").should("not.exist");+ cy.findByTestId("data-app-list-item-second-app").should("exist");});- // The API tells the same story: exactly the two apps, and only the good one serves a bundle.cy.request("GET", "/api/apps").then(({ body: apps }) => {expect(apps.map((app: { name: string }) => app.name).sort()).to.deep.eq([- "broken-bundle","good",+ "second-app",]);});cy.request("/api/apps/good/bundle").its("status").should("eq", 200);-- cy.request({- url: "/api/apps/broken-bundle/bundle",- failOnStatusCode: false,- }).then(({ status, body }) => {- expect(status).to.eq(404);- expect(body).to.deep.eq({ error: "Bundle not synced yet" });- });-- for (const slug of ["bad-config", "no-config"]) {- cy.request({- url: `/api/apps/${slug}/bundle`,- failOnStatusCode: false,- }).then(({ status, body }) => {- expect(status).to.eq(404);- expect(body).to.eq("Not found.");- });- }-- // And what a user opening the broken app sees: its metadata loads (the app- // exists), the host frames it, and the bundle 404 surfaces from inside the- // iframe as the "isn't ready yet" screen — driven by the real pull, no mocks.- // (The not-found screen for a missing app is covered in viewing.cy.spec.ts.)- cy.visit("/apps/broken-bundle");- H.main()- .findByText(/isn.t ready yet/i, { timeout: 30000 })- .should("be.visible");});- it("prunes an app whose directory is removed from the repo on the next sync", () => {+ it("removes an app whose directory is removed from the repo on the next sync", () => {H.copySyncedCollectionFixture();H.copySyncedDataAppsFixture();H.commitToRepo("Add data apps");H.configureGitAndPullChanges("read-write");- // Both apps are materialized from the first pull.- cy.request("GET", "/api/apps").then(({ body: apps }) => {- expect(apps.map((app: { name: string }) => app.name).sort()).to.deep.eq([- "broken-bundle",- "good",- ]);- });-- // Delete the good app's directory from the repo and sync again. The connected- // repo is the source of truth, so the app must be pruned — not left serving.cy.exec(`rm -rf -- "${H.LOCAL_GIT_PATH}/data_apps/good"`);H.commitToRepo("Remove the good app from the repo");H.configureGitAndPullChanges("read-write");- // `good` is gone; `broken-bundle` (still in the repo) survives.cy.request("GET", "/api/apps").then(({ body: apps }) => {expect(apps.map((app: { name: string }) => app.name)).to.deep.eq([- "broken-bundle",+ "second-app",]);});cy.request({ url: "/api/apps/good", failOnStatusCode: false }).its("status").should("eq", 404);-- // The admin list reflects the removal.- cy.visit("/admin/settings/apps");- cy.findByTestId("admin-layout-content").within(() => {- cy.findByTestId("data-app-list-item-broken-bundle").should("exist");- cy.findByTestId("data-app-list-item-good").should("not.exist");- });});});diff --git a/e2e/test/scenarios/data-apps/user-access.cy.spec.ts b/e2e/test/scenarios/data-apps/user-access.cy.spec.tsindex d5feefe605f..ec032ee8c2b 100644--- a/e2e/test/scenarios/data-apps/user-access.cy.spec.ts+++ b/e2e/test/scenarios/data-apps/user-access.cy.spec.ts@@ -352,10 +352,17 @@ describe("scenarios > data apps > user access (EMB-2328)", () => {cy.task("writeDataAppFiles", {files: {[`${H.LOCAL_GIT_PATH}/data_apps/${SYNCED_APP_SLUG}/data_app.yaml`]: [+ "version: 1","name: Good App",+ `slug: ${SYNCED_APP_SLUG}`,"path: ./index.js","allowed_hosts:",` - ${ALLOWED_HOST}`,+ "entity_id: Ioxf30LzIQCGwbCNtaG62",+ "serdes/meta:",+ "- model: DataApp",+ " id: Ioxf30LzIQCGwbCNtaG62",+ ` label: ${SYNCED_APP_SLUG}`,"",].join("\n"),},diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/README.md b/enterprise/backend/src/metabase_enterprise/data_apps/README.mdindex 00a8f97f5bd..ff1ab058675 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/README.md+++ b/enterprise/backend/src/metabase_enterprise/data_apps/README.md@@ -1,125 +1,85 @@# Data apps (backend)A **data app** is a JS bundle, authored in a git repository, that Metabase serves and runs inside a-Near Membrane sandbox in the browser. This README covers the backend half: how apps get from the-repository into the app DB, and how they're served. The sandbox, the app runtime, and the host+Near Membrane sandbox in the browser. This README covers the backend half: how apps are stored, created,+serialized, and served. The sandbox, the app runtime, and the hostiframe live on the frontend and are out of scope here.-## Where apps come from+## What an app is-Apps are **not uploaded**. A single repository is connected to Metabase through remote-sync-(Admin → Settings → Remote sync), and each app lives in its own directory under `data_apps/` at the-repo root:+`:model/DataApp` is a regular serdes entity. A row holds the app's manifest fields — `name` (the+slug, served at `/apps/<slug>`), `display_name`, `description`, the data app contract `version`, the+`bundle_path`, and the `allowed_hosts` its sandboxed bundle may `fetch`/XHR — plus the bundle bytes+themselves, cached in the app DB so serving never reads a repository.-```-data_apps/- sales/ # the directory name is the slug: /apps/sales- data_app.yaml- dist/index.js # the built bundle, committed- ops/- data_app.yaml- dist/app.js-```+The model's hooks hold its invariants: every write is normalized and validated against the column+schemas in `schema.clj`, `bundle_hash` always matches `bundle`, an inserted app gets the collection+and permission group it owns (see Permissions), and a deleted one loses them. Its default fields+leave the bundle out, so listing apps never drags bundles out of the DB; `db.clj`'s+`data-app-bundle` reads it explicitly.-`data_app.yaml` declares the display name and the bundle path relative to the app's own directory,-plus three optional fields: a one-line `description` shown beside the name in the admin UI, the-`version` of the data app contract the app was built for (1 when absent; an app below the version-this Metabase serves is outdated — badged for admins, hidden from everyone else, and a 409 to open),-and the origins the sandboxed bundle may `fetch`/XHR. See `config.clj` for the format and its-validation.+An app with a lower `version` than this Metabase serves is outdated — badged for admins, hidden+from everyone else, and a 409 to open.-**The directory name is the slug.** Nothing in the config declares it. This is what makes slug-collisions structurally impossible — a repo can't hold two `data_apps/sales` directories, and-discovery takes at most one config per directory.+## Serialization-## The sync pipeline--Data apps have no sync of their own; they ride the remote-sync import. Every pull — the manual-"Pull changes" button, the auto-import poll, or startup — ends in-`remote-sync.impl/materialize-data-apps!`, which adapts the imported snapshot to plain reader fns-and calls `data-apps.sync/sync-from-snapshot!`.+A serialized app is a `data_app.yaml` in its own directory under `data_apps/`, with its bundle as a+plain file next to it at its `path`:```-remote-sync import → snapshot → sync-from-snapshot! → discover configs- → materialize each app- → prune apps absent from the repo+data_apps/+ sales/+ data_app.yaml # serdes/meta, entity_id, slug, name, description, version, path, allowed_hosts+ dist/index.js # the bundle```-`data_apps/` is deliberately **not** a serdes path, so apps are invisible to the serialization-layer and are counted separately in the pull summary (see `fold-data-app-changes` in-`remote_sync/impl.clj` — without it, a pull whose only change is a data app reports "no changes").--Apps are synced independently. Each app's row and its resource links commit together, and pruning-runs in its own transaction. A database or resource setup error rolls back that app's update, then-records `sync_error` separately.--## The repository is the source of truth--A sync **upserts every app it finds and deletes every row whose directory is gone.** The-consequences are worth stating explicitly, because they're the questions that come up:--| Event | What happens to apps |-| ----------------------------------- | ------------------------------------------------------------------------- |-| App directory removed from the repo | Row and cached bundle deleted on the next sync |-| Repo switched to a different one | Previous repo's apps are absent from the new snapshot, so they're dropped |-| Repo unlinked | Nothing — unlinking runs no sync, so apps survive |-| Repo has no `data_apps/` at all | All apps removed |-| Failed clone/fetch | Nothing — that throws before a snapshot exists, so deletion never fires |--Pruning is by **directory presence**, not by successful parse. An app whose directory is still-there but whose `data_app.yaml` is momentarily broken keeps its row and its last-good bundle, and is-marked with a `sync_error` — see failure isolation below. Local `enabled` state does not protect a-row: an app disabled in the admin UI and then deleted from the repo is still removed. Draft rows —-created via `POST /api/apps/:slug/draft` ahead of an app's first import — are the one exemption:-pruning skips them until a sync materializes the app and clears the flag.--## Failure isolation--A sync must never take a working app offline, and one bad app must never abort the others. So-failures are recorded per app rather than thrown:+The YAML keeps the keys a hand-written manifest uses: `slug` is the `name` column, `name` the+`display_name`, and `path` the `bundle_path`. The bundle travels as a serdes *resource file*: the+entity carries it in `:serdes/resources` on export, the storage writers put it next to the YAML,+and ingestion reads the paths `serdes/resource-paths` returns back in. A resource path must stay+inside the entity's directory.-- **Malformed `data_app.yaml`** — collected into `:config-errors`. An app that already has a row is- marked with `sync_error` (so the UI shows it as failed rather than silently presenting the- last-good bundle as freshly synced); an app that has no row yet is simply not materialized.-- **Bundle missing or over the size cap** — the row's metadata is still upserted with `sync_error`- set, and the previously cached bundle is kept.-- **Anything thrown at all** — `sync-from-snapshot!` catches and logs it, returning nil, so a- data-app failure can't break the surrounding remote-sync import.+Only the manifest is serialized. `enabled` is admin-owned and never leaves the instance; the+collection, permission group, and `table_ids` are server-managed; `bundle_hash` is recomputed from+the bundle on import. Drafts are not exported.-Because the bundle is cached in the app DB rather than read from the repo per request, a broken sync-degrades to "the app still serves its last good bundle, and the admin sees why it's stale."+An import matches an app by `entity_id`, falling back to its slug so it takes over a draft, and+reasserts the app's resources. It is a no-op without the `:data-apps` feature.-## What survives a sync+Remote sync treats data apps like any other entity, globally rather than per collection. Because+the bundle is a separate file, a pull that changes only a bundle, or an export that touches an app,+takes the full rather than the incremental path. An app's directory also holds its source, which+serialization doesn't own, so exports replace only the YAML and resource files in `data_apps/`.-`enabled` is admin-owned and is never written by a sync — an app disabled in the admin UI stays-disabled across pulls, and new rows get the DB default of true.+## Drafts-`:changed` accounting deliberately ignores `last_synced_sha` / `last_synced_at`: re-syncing-identical content at a new commit is not a change, so the pull summary reports real edits only.+`POST /api/apps/:slug/draft` reserves a slug and creates the app's resources before the app itself+exists, so its resources can be prepared ahead of time. Creating the app fills the draft.## ServingRoutes are mounted at `/api/apps` (`api.clj`). Not `/app/*` — the server reserves that for staticassets (`metabase.server.routes/static-files-handler`).-- `GET /api/apps` — list; `?available=true` filters to enabled apps with no sync error.+- `GET /api/apps` — list; `?available=true` filters to enabled apps that aren't drafts.- `GET /api/apps/:slug` — metadata for one enabled app.- `GET /api/apps/:slug/bundle` — the cached bytes, with a content-hash ETag and `If-None-Match`→ 304. Carries `X-Metabase-Data-App-Allowed-Hosts`, which the iframe reads to configure itssandbox fetch allowlist.- `GET /api/apps/sandbox-host` — the empty document loaded as the Near-Membrane realm iframe,carrying the CSP that confines `'unsafe-eval'` to that realm.-- `PUT /api/apps/:slug` — toggle `enabled` (superuser).+- `POST /api/apps` — create an app from its manifest fields and bundle text, filling a draft with+ the same slug (superuser).+- `PUT /api/apps/:slug` — update manifest fields or the bundle, or toggle `enabled` (superuser).- `DELETE /api/apps/:slug` — drop a row, its bundle, and its owned resources (superuser).-- `POST /api/apps/:slug/draft` — create or reuse a draft row with its resources before the app's- first import (superuser).+- `POST /api/apps/:slug/draft` — create or reuse a draft row with its resources (superuser).- `POST /api/apps/:slug/query` — resolve an authored query definition into a serializableMetabase query plus the table IDs it touches (superuser).- `GET /api/apps/repo-status` — whether a repo is connected (superuser).Responses are field-filtered by role: superusers get full metadata, everyone else gets `name` and`display_name` only. The bundle blob is never serialized into JSON, and metadata reads go through-`data-apps.db`'s non-blob helpers so listing apps doesn't drag the bundles out of the DB.+the model's default fields, which leave the bundle out, so listing apps doesn't drag the bundles out of the DB.`csp.clj` exposes an app's `allowed_hosts` to the core security middleware through a `defenterprise`hook, which drives the `connect-src` of the iframe document's CSP. It's a separate namespace so the@@ -127,8 +87,8 @@ middleware's lookup doesn't pull in route code.## Permissions-Each app owns two server-managed resources (`resources.clj`), created on draft or first import and-reasserted on every sync: a **collection** holding the copies the app is served from (saved+Each app owns two server-managed resources (`resources.clj`), created with the app (or its draft)+and reasserted on every import: a **collection** holding the copies the app is served from (savedquestions, action models, table-sourced metrics) and a **permissions group** its users belong to.The group is set database-level `view-data :blocked` on every database, so it grants **no data@@ -153,10 +113,11 @@ status.| Namespace | Responsibility || --------------------- | --------------------------------------------------------------------------------------------------- |-| `sync.clj` | Discovery, materialization, pruning, drafts. The entry point remote-sync calls. |-| `config.clj` | `data_app.yaml` parsing and validation; the `data_apps/` layout constants. |+| `apps.clj` | Creating apps and drafts; the connected repository's URL. |+| `config.clj` | The serialized layout and data app contract version constants. |+| `schema.clj` | Column schemas, with the normalization and validation every write goes through. || `api.clj` | The `/api/apps` endpoints, bundle serving, ETag handling. || `resources.clj` | Lifecycle of the app-owned collection and permission group: creation, view-data blocking, deletion. |-| `models/data_app.clj` | The `:model/DataApp` Toucan model, permissions, blob coercion. |+| `models/data_app.clj` | The `:model/DataApp` Toucan model: hooks, permissions, default fields, serialization. || `csp.clj` | `allowed_hosts` lookup for the core CSP middleware. || `init.clj` | Loads the above so endpoints, models, and hooks register. |diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/api.clj b/enterprise/backend/src/metabase_enterprise/data_apps/api.cljindex 54c3ddb94d1..f7c449afefd 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/api.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/api.clj@@ -1,22 +1,24 @@(ns metabase-enterprise.data-apps.api- "Data-app endpoints, mounted at `/api/apps`. Serves the bundles materialized by- [[metabase-enterprise.data-apps.sync]]; see `README.md` in this directory for- where apps come from and how the pieces fit together.+ "Data-app endpoints, mounted at `/api/apps`: create, read, update, and delete data apps, and serve their+ bundles. See `README.md` in this directory for how the pieces fit together.Note: we can't use `/app/...` for the frontend or `/api/app/...` for the APIbecause Metabase's server reserves `/app/*` for serving static assets (see`metabase.server.routes/static-files-handler`)."(:require[clojure.string :as str]+ [metabase-enterprise.data-apps.apps :as data-apps.apps][metabase-enterprise.data-apps.config :as data-app.config][metabase-enterprise.data-apps.db :as data-apps.db]+ [metabase-enterprise.data-apps.models.data-app :as data-app][metabase-enterprise.data-apps.query-definition :as query-definition]- [metabase-enterprise.data-apps.sync :as data-app.sync]+ [metabase-enterprise.data-apps.schema :as data-apps.schema][metabase-enterprise.data-apps.user-access :as data-app.user-access][metabase.api-scope.data-app :as api-scope][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.api.routes.common :refer [+auth]]+ [metabase.events.core :as events][metabase.lib-be.core :as lib-be][metabase.lib-be.schema :as lib-be.schema][metabase.lib.core :as lib]@@ -69,7 +71,7 @@origin (conj origin))))(defn- repo-status []- (let [url (data-app.sync/repo-url)]+ (let [url (data-apps.apps/repo-url)]{:configured (some? url):url url}))@@ -99,12 +101,29 @@[:table_ids [:sequential ms/PositiveInt]][:has_user_permission_warnings {:optional true} :boolean][:bundle_hash [:maybe :string]]- [:last_synced_sha [:maybe :string]]- [:last_synced_at [:maybe :any]]- [:sync_error [:maybe :string]][:created_at :any][:updated_at :any]])+(def ^:private CreateDataAppRequest+ [:map {:closed true}+ [:name ::data-apps.schema/slug]+ [:display_name ::data-apps.schema/display-name]+ [:description {:optional true} ::data-apps.schema/description]+ [:version {:optional true} ::data-apps.schema/version]+ [:bundle_path ::data-apps.schema/bundle-path]+ [:allowed_hosts {:optional true} ::data-apps.schema/allowed-hosts]+ [:bundle ::data-apps.schema/bundle-text]])++(def ^:private UpdateDataAppRequest+ [:map {:closed true}+ [:enabled {:optional true} :boolean]+ [:display_name {:optional true} ::data-apps.schema/display-name]+ [:description {:optional true} ::data-apps.schema/description]+ [:version {:optional true} ::data-apps.schema/version]+ [:bundle_path {:optional true} ::data-apps.schema/bundle-path]+ [:allowed_hosts {:optional true} ::data-apps.schema/allowed-hosts]+ [:bundle {:optional true} ::data-apps.schema/bundle-text]])+(def ^:private PublicDataAppResponse[:map {:closed true}[:name ms/NonBlankString]@@ -253,12 +272,11 @@app)(api.macros/defendpoint :get "/" :- [:sequential [:or DataAppResponse PublicDataAppResponse]]- "List the data apps provided by the connected repository. Pass `available=true`- to return only enabled apps without sync errors. An outdated app is never+ "List the data apps. Pass `available=true` to return only enabled apps that aren't drafts. An outdated app is neveravailable, and otherwise listed only to superusers, who see it badged."[_route-params{:keys [available]} :- [:map {:closed true} [:available {:optional true} [:maybe :boolean]]]]- (let [apps (->> (data-apps.db/non-blob-data-apps available)+ (let [apps (->> (data-apps.db/data-apps available)(remove #(and (or available (not api/*is-superuser?*))(data-app.config/outdated? %)))(mapv api/read-check))@@ -266,28 +284,51 @@(data-app.user-access/groups-with-permission-warnings apps))](mapv (partial data-app-list-response warning-group-ids) apps)))+;; NOTE on the `slug-regex` constraint: the default path-param matcher allows+;; slashes inside a segment, so `/:slug` would otherwise swallow `/x/bundle`.+;; The regex also excludes the literal `repo-status` sub-route above.+(defn- with-bundle+ "The DataApp `changes` with their `bundle` text as the bytes the app is served."+ [changes]+ (cond-> changes+ (contains? changes :bundle) (update :bundle data-app/file->bundle)))++(defn- write-check-data-app+ "The DataApp named `slug` without its bundle, checked for existence and write permissions."+ [slug]+ (api/write-check (api/check-404 (data-apps.db/data-app-by-slug slug))))++(api.macros/defendpoint :post "/" :- DataAppResponse+ "Create a data app from its manifest fields and bundle. A draft with the same slug becomes the app."+ [_route-params+ _query-params+ body :- CreateDataAppRequest]+ (api/create-check :model/DataApp body)+ (let [app (data-apps.db/data-app (data-apps.apps/create-app! (with-bundle body)))]+ (events/publish-event! :event/data-app-create {:object app :user-id api/*current-user-id*})+ (data-app-response app)))+;; NOTE on the `slug-regex` constraint: the default path-param matcher allows;; slashes inside a segment, so `/:slug` would otherwise swallow `/x/bundle`.;; The regex also excludes the literal `repo-status` sub-route above.(api.macros/defendpoint :put ["/:slug" :slug slug-regex] :- DataAppResponse- "Enable or disable a single data app. Disabled apps are not served."+ "Update a data app's manifest fields or bundle, or enable or disable it. Disabled apps are not served."[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]_query-params- {:keys [enabled]} :- [:map {:closed true} [:enabled :boolean]]]- (api/check-superuser)- (let [app (api/check-404 (data-apps.db/non-blob-data-app-by-slug slug))]- (data-apps.db/update-data-app! (:id app) {:enabled enabled})- (data-app-response (data-apps.db/non-blob-data-app (:id app)))))+ changes :- UpdateDataAppRequest]+ (let [app (write-check-data-app slug)]+ (when (seq changes)+ (data-apps.db/update-data-app! (:id app) (with-bundle changes)))+ (let [app (data-apps.db/data-app (:id app))]+ (events/publish-event! :event/data-app-update {:object app :user-id api/*current-user-id*})+ (data-app-response app))))(api.macros/defendpoint :delete ["/:slug" :slug slug-regex] :- :nil- "Remove a single data app (its row and cached bundle). Intended for clearing out- apps left behind by a repository that is no longer connected: while a repo is- connected, an app still in it is re-materialized by the next sync, and one no- longer in it is pruned by that sync anyway."+ "Delete a data app, its bundle, and the collection and permission group it owns."[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]]- (api/check-superuser)- ;; The delete returns the row count; a 0 means the slug wasn't there → 404.- (api/check-404 (pos? (data-apps.db/delete-data-app-by-slug! slug)))+ (let [app (write-check-data-app slug)]+ (data-apps.db/delete-data-app! (:id app))+ (events/publish-event! :event/data-app-delete {:object app :user-id api/*current-user-id*}));; a `nil` body is rendered as a 204; matches the `:- :nil` response schema;; above (returning `generic-204-no-content` would fail that validation).nil)@@ -297,14 +338,13 @@[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]_query-params{table-ids :table_ids} :- TableDependenciesRequest]- (api/check-superuser)- (let [app (api/check-404 (data-apps.db/non-blob-data-app-by-slug slug))+ (let [app (write-check-data-app slug)table-ids (vec (sort table-ids))](api/check-400 (= (set table-ids)(data-apps.db/existing-table-ids table-ids))(tru "One or more tables do not exist."))(data-apps.db/update-data-app! (:id app) {:table_ids table-ids})- (data-app-response (data-apps.db/non-blob-data-app (:id app)))))+ (data-app-response (data-apps.db/data-app (:id app)))))(api.macros/defendpoint :post ["/:slug/user-permission-warnings" :slug slug-regex]:- [:sequential PermissionWarning]@@ -312,8 +352,7 @@[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]_query-params{user-ids :user_ids} :- PermissionWarningsRequest]- (api/check-superuser)- (let [app (api/check-404 (data-apps.db/non-blob-data-app-by-slug slug))+ (let [app (write-check-data-app slug)users (data-apps.db/users-for-permission-warnings user-ids)](api/check-404 (= (count users) (count user-ids)))(api/check-400 (every? :is_active users)@@ -363,8 +402,7 @@[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]_query-paramsquery-def :- ::query-definition/query-definition]- (api/check-superuser)- (api/check-404 (data-apps.db/non-blob-data-app-by-slug slug))+ (write-check-data-app slug)(let [{source-type :type, table-id :id} (get-in query-def [:stages 0 :source])_ (api/check-400 (= (keyword source-type) :table)"Data app query definitions must use a table source.")@@ -385,8 +423,7 @@[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]_query-params{dataset-queries :dataset_queries} :- QueryTableDependenciesRequest]- (api/check-superuser)- (api/check-404 (data-apps.db/non-blob-data-app-by-slug slug))+ (write-check-data-app slug){:table_ids (->> dataset-queries(mapcat (fn [dataset-query](-> (lib-be/application-database-metadata-provider (:database dataset-query))@@ -397,13 +434,11 @@vec)})(api.macros/defendpoint :post ["/:slug/draft" :slug slug-regex] :- DataAppResponse- "Create or reuse a data app draft before its first repository import."- [{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]]- (api/check-superuser)- (api/check-400 (data-app.config/valid-slug? slug)- "Data app draft slugs must use lowercase letters, numbers, and dashes.")- (data-app.sync/ensure-draft! slug)- (data-app-response (data-apps.db/non-blob-data-app-by-slug slug)))+ "Create or reuse a data app draft, which reserves a slug and the app's resources before the app is created."+ [{:keys [slug]} :- [:map {:closed true} [:slug ::data-apps.schema/slug]]]+ (api/create-check :model/DataApp {:name slug})+ (data-apps.apps/ensure-draft! slug)+ (data-app-response (data-apps.db/data-app-by-slug slug)));; Not tagged `data-apps:base`, though the bundle route below is — which looks backwards until;; you place the two callers. `DataAppView` fetches this metadata on the *host* page to decide@@ -412,7 +447,7 @@(api.macros/defendpoint :get ["/:slug" :slug slug-regex] :- [:or DataAppResponse PublicDataAppResponse]"Fetch metadata for a single enabled data app by its slug."[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]]- (let [app (read-check-data-app (data-apps.db/enabled-non-blob-data-app-by-slug slug))]+ (let [app (read-check-data-app (data-apps.db/enabled-data-app-by-slug slug))](data-app-response (cond-> app (not api/*is-superuser?*) check-not-outdated))))(api.macros/defendpoint :get ["/:slug/bundle" :slug slug-regex] :- :any@@ -426,7 +461,7 @@respondraise](try- (let [row (check-not-outdated (read-check-data-app (data-apps.db/enabled-non-blob-data-app-by-slug slug)))+ (let [row (check-not-outdated (read-check-data-app (data-apps.db/enabled-data-app-by-slug slug)))hash (:bundle_hash row)etag (some->> hash (format "\"%s\""))](conddiff --git a/enterprise/backend/src/metabase_enterprise/data_apps/apps.clj b/enterprise/backend/src/metabase_enterprise/data_apps/apps.cljnew file mode 100644index 00000000000..02f6bc155e7--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/data_apps/apps.clj@@ -0,0 +1,63 @@+(ns metabase-enterprise.data-apps.apps+ "Creating data apps and their drafts, and the connected repository's URL."+ (:require+ [clojure.string :as str]+ [metabase-enterprise.data-apps.db :as data-apps.db]+ [metabase-enterprise.data-apps.resources :as data-app.resources]+ [metabase.api.common :as api]+ [metabase.settings.core :as setting]+ [metabase.util.i18n :refer [tru]]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^:private draft-bundle-path+ "The bundle path of a draft, until the app is created."+ "dist/index.js")++(defn repo-url+ "The connected remote-sync repository URL, or nil when none is configured. Read+ by keyword so this OSS namespace has no compile-time dependency on the+ enterprise remote-sync module; returns nil when that module (and its setting)+ isn't loaded."+ []+ (let [url (try (setting/get :remote-sync-url) (catch Throwable _ nil))]+ (when-not (str/blank? url)+ url)))++(defn- create-draft!+ [slug]+ (let [app (t2/with-transaction [_conn]+ (when-not (data-apps.db/data-app-exists? slug)+ (data-apps.db/insert-data-app!+ {:name slug+ :display_name slug+ :bundle_path draft-bundle-path+ :draft true}))+ (data-apps.db/data-app-by-slug slug))]+ (data-app.resources/ensure-resources! app)))++(defn ensure-draft!+ "Create a data app draft when needed and ensure its permission resources.+ Creating the app later fills the same row with its manifest and bundle."+ [slug]+ (try+ (create-draft! slug)+ (catch Throwable e+ (if (data-apps.db/data-app-exists? slug)+ (create-draft! slug)+ (throw e)))))++(defn create-app!+ "Create the data app `row`, filling the draft with its slug if there is one, and return its ID. Throws a 409+ when a data app that isn't a draft already has the slug."+ [{slug :name :as row}]+ (t2/with-transaction [_conn]+ (let [existing (data-apps.db/data-app-by-slug slug)+ row (assoc row :draft false)]+ (api/check (or (nil? existing) (:draft existing))+ [409 (tru "A data app with this slug already exists.")])+ (if existing+ (do (data-apps.db/update-data-app! (:id existing) (dissoc row :name))+ (:id existing))+ (data-apps.db/insert-data-app-returning-pk! row)))))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/config.clj b/enterprise/backend/src/metabase_enterprise/data_apps/config.cljindex b549647f49b..38fa86b1889 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/config.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/config.clj@@ -1,29 +1,19 @@(ns metabase-enterprise.data-apps.config- "Parsing + validation of a per-app `data_app.yaml`, which sits in the app's own- directory under `data_apps/` (see `README.md` in this directory for the layout):+ "Constants of a data app's manifest, which a serialized app keeps in the `data_app.yaml` in its own directory+ under `data_apps/` (see `README.md` in this directory for the layout); the columns' schemas are in+ [[metabase-enterprise.data-apps.schema]]:- name: Sales dashboard # display name+ slug: sales # the slug the app is served at (column `name`)+ name: Sales dashboard # display name (column `display_name`)description: Pipeline … # optional — one line on what the app doesversion: 1 # optional — data app contract version, 1 when absent- path: dist/index.js # bundle path, relative to this app's directory+ path: dist/index.js # bundle path, relative to this app's directory (column `bundle_path`)allowed_hosts: # optional — origins the sandboxed bundle may fetch/XHR- https://api.example.com- - https://*.internal.acme.com"- (:require- [clojure.java.io :as io]- [clojure.string :as str]- [metabase.util :as u]- [metabase.util.i18n :refer [tru]]- [metabase.util.yaml :as yaml])- (:import- (java.io ByteArrayInputStream)))+ - https://*.internal.acme.com")(set! *warn-on-reflection* true)-(def config-file-name- "Name of the per-app config file (used in error messages)."- "data_app.yaml")-(def apps-dir"Directory at the repo root that holds one subdirectory per data app.""data_apps")@@ -34,180 +24,7 @@listed only to admins, badged, and never opened."1)-(def ^:private initial-app-version- "The `version` of an app whose manifest declares none: every such app predates the- field, so this stays 1 while [[supported-app-version]] moves on."- 1)-(defn outdated?"Whether `app` was built for a contract version older than [[supported-app-version]]."[{:keys [version]}](< version supported-app-version))--(defn- parse-version- "Validate the optional `version`: a positive whole number, [[initial-app-version]]- when absent. Throws a 400 for anything else rather than coercing it, since a- string or a decimal here is a typo, not a version."- [raw ^String dir]- (cond- (nil? raw)- initial-app-version-- (and (integer? raw) (pos? raw))- (long raw)-- :else- (throw (ex-info (tru "{0}/{1}: \"version\" must be a positive whole number, such as 1." dir config-file-name)- {:status-code 400}))))--(def ^:private slug-pattern- "Lowercase letters/numbers separated by single dashes. An app *directory* must- match this: the name is used verbatim as the slug, never normalized. Folding- case here would reintroduce the collisions the directory naming rules out —- git happily holds both `data_apps/Sales` and `data_apps/sales`."- #"[a-z0-9]+(?:-[a-z0-9]+)*")--(def ^:private reserved-slugs- "Slugs that collide with literal `/api/apps/*` sub-routes (see the API's- `slug-regex`). An app with one of these would sync but be unreachable, so we- reject it during parsing."- #{"repo-status" "sandbox-host"})--(defn- normalize-path- "Trim and drop a leading `./` so the path is relative to the app directory."- [p]- (-> (str p) str/trim (str/replace #"^\./" "")))--(defn- normalize-description- "Trim and fold whitespace runs, so a YAML block scalar still stores as one line."- [d]- (-> (str d) str/trim (str/replace #"\s+" " ")))--(def ^:private max-description-chars- "Cap on `description`. Checked here, not by the column type, so an over-long one fails just its own app."- 255)--(defn- parse-description- "Validate + normalize the optional one-line `description`. Returns nil when it's- absent or blank; throws a 400 when it's present but not a string — a YAML list or- mapping would otherwise `str`-ify into Clojure collection syntax and sync as-is —- or when it exceeds [[max-description-chars]]."- [raw ^String dir]- (when (some? raw)- (when-not (string? raw)- (throw (ex-info (tru "{0}/{1}: \"description\" must be a one-line string." dir config-file-name)- {:status-code 400})))- (let [d (not-empty (normalize-description raw))]- (when (and d (> (count d) max-description-chars))- (throw (ex-info (tru "{0}/{1}: \"description\" is a one-line summary — it must be {2} characters or fewer."- dir config-file-name max-description-chars)- {:status-code 400})))- d)))--(def ^:private allowed-host-re- "A single `allowed_hosts` entry: an origin the app's sandboxed bundle may- `fetch`/XHR — scheme + host, an optional `*.` subdomain wildcard, and an- optional port. No path/query/`*`/non-http(s) scheme: it names an- origin, e.g. `https://api.example.com` or `https://*.internal.acme.com`."- #"(?i)https?://(\*\.)?[a-z0-9-]+(\.[a-z0-9-]+)*(:\d+)?")--(defn- normalized-allowed-host- "Trim, lowercase, and drop trailing slashes from one `allowed_hosts` entry."- [entry]- (some-> entry str str/trim u/lower-case-en (str/replace #"/+$" "")))--(defn- parse-allowed-host- "Normalize + validate a single `allowed_hosts` entry (see [[allowed-host-re]]).- Throws an `ex-info` with `:status-code` 400 when it isn't an origin-only value."- [entry ^String dir]- (let [s (normalized-allowed-host entry)]- (when-not (and (seq s) (re-matches allowed-host-re s))- (throw (ex-info (tru "{0}/{1}: \"{2}\" is not a valid allowed_hosts entry — use an origin like https://api.example.com or https://*.example.com."- dir config-file-name (str entry))- {:status-code 400})))- s))--(defn- parse-allowed-hosts- "Validate + normalize the optional `allowed_hosts` list from a parsed config.- Each entry must be an origin-only value (see [[allowed-host-re]]); returns a- lowercased, de-duplicated vector, or `[]` when the key is absent. Throws an- `ex-info` with `:status-code` 400 on a non-list value or an invalid entry."- [parsed ^String dir]- (let [raw (:allowed_hosts parsed)]- (cond- (nil? raw) []- (not (sequential? raw))- (throw (ex-info (tru "{0}/{1}: \"allowed_hosts\" must be a list." dir config-file-name)- {:status-code 400}))- :else- (->> raw- (map #(parse-allowed-host % dir))- distinct- vec))))--(defn- path-traversal? [path]- (some #(= ".." %) (str/split path #"/")))--(defn- parse-yaml [^bytes bytes ^String dir]- (try- (with-open [r (io/reader (ByteArrayInputStream. bytes) :encoding "UTF-8")]- (yaml/parse-stream r))- (catch Exception e- (throw (ex-info (tru "Could not parse {0}/{1}: {2}" dir config-file-name (ex-message e))- {:status-code 400})))))--(defn dir-slug- "The app's slug: the name of its directory (`data_apps/sales` -> `sales`). Used- verbatim, never normalized. Public so discovery can label an app by its- directory even when its `data_app.yaml` fails to parse (so a transiently broken- config isn't treated as a removal)."- [^String dir]- (subs dir (inc (str/last-index-of dir "/"))))--(defn valid-slug?- "Whether `slug` can name a data app directory and API route."- [slug]- (and (string? slug)- (re-matches slug-pattern slug)- (not (contains? reserved-slugs slug))))--(defn parse-app-config- "Parse the bytes of one `data_app.yaml` from the app directory `dir` (e.g.- `data_apps/sales`) into `{:slug ..., :display_name ..., :description ...,- :version ..., :path ..., :allowed_hosts [...]}`. The slug is the directory's- name; `path` is relative to the directory; `:description` is an optional- one-liner (nil when absent or blank, capped at [[max-description-chars]]);- `:version` is the contract version the app was built for (1 when absent);- `:allowed_hosts` is a (possibly empty) vector of origins the sandboxed bundle- may reach. Throws an- `ex-info` with `:status-code` 400 on malformed or incomplete content — including- a directory whose name isn't a usable slug, since that app has no URL to be served at."- [^bytes bytes ^String dir]- (let [parsed (parse-yaml bytes dir)- slug (dir-slug dir)- name (some-> (:name parsed) str str/trim not-empty)- description (parse-description (:description parsed) dir)- version (parse-version (:version parsed) dir)- path (some-> (:path parsed) normalize-path not-empty)- allowed-hosts (parse-allowed-hosts parsed dir)]- (when-not (re-matches slug-pattern slug)- (throw (ex-info (tru "{0}: the app directory''s name is its slug, so it must be lowercase letters, numbers, and dashes." dir)- {:status-code 400})))- (when (contains? reserved-slugs slug)- (throw (ex-info (tru "{0}: \"{1}\" is a reserved slug — rename the directory." dir slug)- {:status-code 400})))- (when-not name- (throw (ex-info (tru "{0}/{1}: \"name\" is required." dir config-file-name)- {:status-code 400})))- (when-not path- (throw (ex-info (tru "{0}/{1}: \"path\" is required." dir config-file-name)- {:status-code 400})))- (when (path-traversal? path)- (throw (ex-info (tru "{0}/{1}: \"path\" must not contain \"..\"." dir config-file-name)- {:status-code 400})))- {:slug slug- :display_name name- :description description- :version version- :path path- :allowed_hosts allowed-hosts}))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/csp.clj b/enterprise/backend/src/metabase_enterprise/data_apps/csp.cljindex fb49d3dd750..ec8f7de99c1 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/csp.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/csp.clj@@ -12,5 +12,5 @@CSP `connect-src` so the sandboxed bundle can fetch/XHR those origins.":feature :data-apps[slug]- (or (:allowed_hosts (data-apps.db/enabled-non-blob-data-app-by-slug slug))+ (or (:allowed_hosts (data-apps.db/enabled-data-app-by-slug slug))[]))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj b/enterprise/backend/src/metabase_enterprise/data_apps/db.cljindex 21dfb1feaf3..9e76e4387ea 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/db.clj@@ -9,47 +9,59 @@[metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay][toucan2.core :as t2]))-(def ^:private non-blob-columns- "Columns to select for normal data-app metadata reads, excluding the raw bundle blob."- [:model/DataApp :id :name :display_name :description :version :bundle_path :enabled :allowed_hosts- :resource_collection_id :permission_group_id :table_ids :draft- :bundle_hash :last_synced_sha :last_synced_at :sync_error- :created_at :updated_at])--(mu/defn non-blob-data-app+(def non-blob-columns+ "Every DataApp column except the raw bundle blob."+ [:id :entity_id :name :display_name :description :version :bundle_path :enabled :allowed_hosts+ :resource_collection_id :permission_group_id :table_ids :draft :bundle_hash :created_at :updated_at])++(def ^:private non-blob-model+ (into [:model/DataApp] non-blob-columns))++(mu/defn data-app"The DataApp with `data-app-id` without its bundle, or nil."[data-app-id :- ms/PositiveInt]- (t2/select-one non-blob-columns :id data-app-id))+ (t2/select-one non-blob-model :id data-app-id))-(mu/defn non-blob-data-app-by-slug+(mu/defn data-app-by-slug"The DataApp named `slug` without its bundle, or nil."[slug :- :string]- (t2/select-one non-blob-columns :name slug))+ (t2/select-one non-blob-model :name slug))++(mu/defn draft-by-slug+ "The draft DataApp named `slug` without its bundle, or nil."+ [slug :- :string]+ (t2/select-one non-blob-model :name slug :draft true))-(mu/defn enabled-non-blob-data-app-by-slug+(mu/defn enabled-data-app-by-slug"The enabled DataApp named `slug` without its bundle, or nil."[slug :- :string]- (t2/select-one non-blob-columns :name slug :enabled true))+ (t2/select-one non-blob-model :name slug :enabled true))-(mu/defn non-blob-data-apps- "Every DataApp without its bundle, ordered by display name; only the enabled, error-free ones when `available?`."+(mu/defn data-apps+ "Every DataApp without its bundle, ordered by display name; only the enabled ones that aren't drafts when+ `available?`."[available? :- [:maybe :boolean]]- (t2/select non-blob-columns+ (t2/select non-blob-model(cond-> {:order-by [[:display_name :asc]]}available? (assoc :where [:and[:= :enabled true]- [:= :sync_error nil]]))))+ [:= :draft false]]))))(mu/defn data-app-bundle"The bundle bytes of the DataApp with `data-app-id`."[data-app-id :- ms/PositiveInt]- (t2/select-one-fn :bundle :model/DataApp :id data-app-id))--(mu/defn data-apps-sync-info- "The sync-relevant columns of every DataApp."- []- (t2/select [:model/DataApp :name :display_name :description :version :allowed_hosts :bundle_path- :bundle_hash :sync_error]))+ (t2/select-one-fn :bundle [:model/DataApp :bundle] :id data-app-id))++(mu/defn reducible-data-apps-with-bundles+ "A reducible of the DataApp rows, bundles included, whose `filter-column` is one of `filter-ids` (every row when+ `filter-column` is nil), ordered ascending by `order-columns`."+ [filter-column :- [:maybe :keyword]+ filter-ids :- [:maybe [:sequential [:maybe [:or :int :string]]]]+ order-columns :- [:maybe [:sequential :keyword]]]+ (t2/reducible-select [:model/DataApp :*]+ (cond-> {}+ filter-column (assoc :where [:in filter-column filter-ids])+ (seq order-columns) (assoc :order-by (mapv (fn [column] [column :asc]) order-columns)))))(defn table-database-id"The database ID for the Table with `table-id`, or nil."@@ -87,37 +99,21 @@[row :- ::data-apps.schema/data-app.update](t2/insert! :model/DataApp row))+(mu/defn insert-data-app-returning-pk! :- ms/PositiveInt+ "Insert the DataApp `row`, returning its ID."+ [row :- ::data-apps.schema/data-app.update]+ (t2/insert-returning-pk! :model/DataApp row))+(mu/defn update-data-app!"Apply `changes` to the DataApp with `data-app-id`."[data-app-id :- ms/PositiveIntchanges :- ::data-apps.schema/data-app.update](t2/update! :model/DataApp :id data-app-id changes))-(mu/defn update-data-app-by-slug!- "Apply `changes` to the DataApp named `slug`."- [slug :- :string- changes :- ::data-apps.schema/data-app.update]- (t2/update! :model/DataApp :name slug changes))--(mu/defn delete-data-app-by-slug!- "Delete the DataApp named `slug`, returning the number deleted."- [slug :- :string]- (t2/delete! :model/DataApp :name slug))--(mu/defn delete-data-apps-not-named!- "Delete non-draft DataApps whose name is not one of `slugs`, returning the number deleted."- [slugs :- [:set :string]]- (t2/delete! :model/DataApp :name [:not-in slugs] :draft false))--(mu/defn delete-all-data-apps!- "Delete every non-draft DataApp, returning the number deleted."- []- (t2/delete! :model/DataApp :draft false))--(defn publish-data-app-drafts!- "Mark drafts named by `slugs` as published."- [slugs]- (t2/update! :model/DataApp :name [:in slugs] :draft true {:draft false}))+(mu/defn delete-data-app!+ "Delete the DataApp with `data-app-id`."+ [data-app-id :- ms/PositiveInt]+ (t2/delete! :model/DataApp :id data-app-id))(defn permission-group"The permission group with `group-id`, or nil."diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/init.clj b/enterprise/backend/src/metabase_enterprise/data_apps/init.cljindex 53251438552..2a02e6ec974 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/init.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/init.clj@@ -1,7 +1,7 @@(ns metabase-enterprise.data-apps.init(:require[metabase-enterprise.data-apps.api]+ [metabase-enterprise.data-apps.apps][metabase-enterprise.data-apps.config][metabase-enterprise.data-apps.csp]- [metabase-enterprise.data-apps.models.data-app]- [metabase-enterprise.data-apps.sync]))+ [metabase-enterprise.data-apps.models.data-app]))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj b/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.cljindex d7b9017dd37..5f89bbe249b 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj@@ -1,14 +1,25 @@(ns metabase-enterprise.data-apps.models.data-app(:require+ [metabase-enterprise.data-apps.config :as data-app.config][metabase-enterprise.data-apps.db :as data-apps.db][metabase-enterprise.data-apps.resources :as data-app.resources]+ [metabase-enterprise.data-apps.schema :as data-apps.schema][metabase.api.common :as api]+ [metabase.events.core :as events]+ [metabase.lib.core :as lib][metabase.models.interface :as mi]+ [metabase.models.serialization :as serdes][metabase.premium-features.core :refer [defenterprise]]+ [metabase.util.i18n :refer [tru]]+ [metabase.util.malli :as mu][methodical.core :as methodical]- [toucan2.core :as t2])+ [toucan2.core :as t2]+ [toucan2.tools.default-fields :as t2.default-fields])(:import- (java.sql Blob)))+ (java.nio.charset StandardCharsets)+ (java.security MessageDigest)+ (java.sql Blob)+ (org.apache.commons.codec.binary Hex)))(set! *warn-on-reflection* true)@@ -33,25 +44,41 @@(doto :model/DataApp(derive :metabase/model)+ (derive :hook/entity-id)(derive :hook/timestamped?))-(defn- default-last-synced-at- "A sync that records `last_synced_sha` happened now unless it says otherwise."+(t2.default-fields/define-default-fields :model/DataApp+ data-apps.db/non-blob-columns)++(events/derive! ::event :metabase/event)+(doseq [e [:event/data-app-create :event/data-app-update :event/data-app-delete]]+ (events/derive! e ::event))++(defn- bytes-hash ^String [^bytes b]+ (let [^MessageDigest md (MessageDigest/getInstance "SHA-256")]+ (Hex/encodeHexString ^bytes (.digest md b))))++(defn- with-bundle-hash+ "The `row` with `bundle_hash` matching the `bundle` it sets."[row](cond-> row- (and (contains? row :last_synced_sha) (not (contains? row :last_synced_at)))- (assoc :last_synced_at (mi/now))))+ (contains? row :bundle) (assoc :bundle_hash (some-> ^bytes (:bundle row) bytes-hash))))++(defn- prepare+ "The DataApp `row` normalized and validated against `schema`, with the hash of any bundle it sets."+ [schema row]+ (->> row+ (lib/normalize schema)+ (mu/validate-throw schema)+ with-bundle-hash))(t2/define-before-insert :model/DataApp[data-app]- (default-last-synced-at data-app))+ (prepare ::data-apps.schema/data-app.insert data-app))(t2/define-before-update :model/DataApp[data-app]- (let [changes (t2/changes data-app)]- (cond-> data-app- (and (contains? changes :last_synced_sha) (not (contains? changes :last_synced_at)))- (assoc :last_synced_at (mi/now)))))+ (merge data-app (some->> (t2/changes data-app) (prepare ::data-apps.schema/data-app.update))));; Reads always see `allowed_hosts` as a vector, never nil — a row synced before;; the column existed has NULL until it's re-synced. Guard on `contains?` so@@ -78,6 +105,10 @@[_model _instance]api/*is-superuser?*)+(t2/define-after-insert :model/DataApp+ [app]+ (merge app (data-app.resources/ensure-resources! app)))+(t2/define-before-delete :model/DataApp[app](data-app.resources/delete-resources! app))@@ -87,6 +118,70 @@[data-app json-generator](next-method (dissoc data-app :bundle) json-generator))+(defn- bundle->file+ "The `bundle` bytes as the text of its file."+ [^bytes bundle]+ (String. bundle StandardCharsets/UTF_8))++(defn file->bundle+ "The `content` of a bundle file as the bytes the app is served."+ ^bytes [^String content]+ (.getBytes content StandardCharsets/UTF_8))++(defn- ingested-bundle-path+ "The normalized bundle path of the ingested data app `ingested`."+ [ingested]+ (lib/normalize ::data-apps.schema/bundle-path (:path ingested)))++(defmethod serdes/make-spec "DataApp"+ [_model-name _opts]+ {:copy [:entity_id :description :version :allowed_hosts]+ :skip [;; admin-owned state of this instance+ :enabled+ ;; set by the import itself+ :draft :bundle_hash+ ;; server-managed resources, recreated on import+ :resource_collection_id :permission_group_id :table_ids]+ :transform {:created_at (serdes/date)+ :name {:as :slug :export identity :import identity}+ :display_name {:as :name :export identity :import identity}+ :bundle_path {:as :path :export identity :import identity}+ :bundle {:as :serdes/resources+ :export-with-context (fn [app _k bundle]+ (if bundle+ {(:bundle_path app) (bundle->file bundle)}+ ::serdes/skip))+ :import-with-context (fn [ingested _k files]+ (let [path (ingested-bundle-path ingested)]+ (file->bundle+ (or (get files path)+ (throw (ex-info (tru "Bundle file \"{0}\" not found." path)+ {:status-code 400}))))))}}+ :defaults {:description nil :version 1 :allowed_hosts []}})++(defmethod serdes/extract-query "DataApp"+ [model-name {:keys [filter-column filter-ids] :as opts}]+ (eduction (remove :draft)+ (data-apps.db/reducible-data-apps-with-bundles filter-column filter-ids+ (serdes/extract-order-columns model-name opts))))++(defmethod serdes/storage-path "DataApp" [app _ctx]+ [{:label data-app.config/apps-dir}+ {:label (:slug app) :key (:entity_id app) :style :slug}+ {:label "data_app"}])++(defmethod serdes/resource-paths "DataApp" [ingested]+ [(ingested-bundle-path ingested)])++(defmethod serdes/load-one! "DataApp"+ [ingested maybe-local]+ (let [local (or maybe-local (data-apps.db/draft-by-slug (:slug ingested)))+ app (serdes/default-load-one! ingested local)]+ (data-apps.db/update-data-app! (:id app) {:draft false})+ (when local+ (data-app.resources/ensure-resources! app))+ app))+(defenterprise data-app-group-ids"The data-app permission groups (those flagged `is_data_app_group`). SSO group sync must never touchtheir membership."diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/resources.clj b/enterprise/backend/src/metabase_enterprise/data_apps/resources.cljindex 43e10fb3656..5c653c5206f 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/resources.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/resources.clj@@ -76,7 +76,7 @@"Create or restore the server-owned permission resources for `app` and return their IDs."[app](perms/with-global-permissions-lock- (let [app (data-apps.db/non-blob-data-app (:id app))+ (let [app (data-apps.db/data-app (:id app))group (permission-group! app)collection (resource-collection! app)](data-apps.db/update-permission-group! (:id group)diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj b/enterprise/backend/src/metabase_enterprise/data_apps/schema.cljindex bd27ba3c16a..ff0c1ccff90 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj@@ -1,9 +1,95 @@(ns metabase-enterprise.data-apps.schema"Malli schemas for the data-apps module."(:require+ [clojure.string :as str]+ [metabase.util :as u][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))+(def max-bundle-bytes+ "Cap on a data app bundle's size (10 MiB)."+ (* 10 1024 1024))++(def ^:private reserved-slugs+ "Slugs that collide with literal `/api/apps/*` sub-routes, so an app with one would be unreachable."+ #{"repo-status" "sandbox-host"})++(defn- decoders+ "Schema properties decoding a value with `f` both in [[metabase.lib.core/normalize]] and at the API boundary."+ [f]+ {:decode/normalize f, :decode/api f})++(defn- trim [x]+ (cond-> x (string? x) str/trim))++(defn- one-line [x]+ (cond-> x (string? x) (-> (str/replace #"\s+" " ") str/trim not-empty)))++(defn- relative-path [x]+ (cond-> x (string? x) (-> str/trim (str/replace #"^\./" ""))))++(defn- origin [x]+ (cond-> x (string? x) (-> str/trim u/lower-case-en (str/replace #"/+$" ""))))++(defn- version-or-1 [x]+ (if (nil? x) 1 x))++(defn- distinct-vec [x]+ (cond-> x (nil? x) vec, (sequential? x) (-> distinct vec)))++(mr/def ::slug+ "A data app's slug: lowercase letters and numbers separated by single dashes, at most 100 characters."+ [:and+ {:error/message "must be lowercase letters, numbers, and dashes"}+ [:string {:max 100}]+ [:re #"\A[a-z0-9]+(?:-[a-z0-9]+)*\z"]+ [:fn {:error/message "must not be a reserved slug"} (complement reserved-slugs)]])++(mr/def ::display-name+ "A data app's display name."+ [:string (merge {:min 1} (decoders trim))])++(mr/def ::description+ "A data app's optional one-line summary, nil when blank."+ [:maybe (decoders one-line) [:string {:max 255}]])++(mr/def ::version+ "The data app contract version an app was built for, 1 when it declares none."+ [:int (merge {:min 1} (decoders version-or-1))])++(mr/def ::bundle-path+ "The path of a data app's bundle, relative to the app's directory and inside it."+ [:and+ (decoders relative-path)+ ms/NonBlankString+ [:fn {:error/message "must be relative to the app's directory and stay inside it"}+ #(and (not (str/starts-with? % "/"))+ (not (str/includes? % "\\"))+ (not-any? #{"" "." ".."} (str/split % #"/" -1)))]+ [:fn {:error/message "must not be the app's data_app.yaml"} #(not= "data_app.yaml" %)]])++(mr/def ::allowed-host+ "An origin a data app's sandboxed bundle may fetch: scheme, host with an optional `*.` subdomain wildcard, and an+ optional port."+ [:re+ (merge {:error/message "must be an origin like https://api.example.com or https://*.example.com"}+ (decoders origin))+ #"\Ahttps?://(\*\.)?[a-z0-9-]+(\.[a-z0-9-]+)*(:\d+)?\z"])++(mr/def ::allowed-hosts+ "The origins a data app's sandboxed bundle may fetch, `[]` when nil."+ [:sequential (decoders {:leave distinct-vec}) ::allowed-host])++(mr/def ::bundle+ "A data app's bundle bytes."+ [:and+ bytes?+ [:fn {:error/message "must be less than 10 MiB"} #(<= (alength ^bytes %) max-bundle-bytes)]])++(mr/def ::bundle-text+ "The text of a data app's bundle file."+ [:string {:max max-bundle-bytes}])+(mr/def ::data-app"A DataApp as selected from the app DB: every column of `:data_app`."[:merge@@ -14,21 +100,28 @@(mr/def ::data-app.update"What an update (or insert) of a DataApp accepts: every column of `:data_app` except `id`, all optional."[:map {:closed true}- [:name {:optional true} [:maybe :string]]- [:display_name {:optional true} [:maybe :string]]- [:allowed_hosts {:optional true} [:maybe [:sequential :string]]]- [:bundle_path {:optional true} [:maybe :string]]- [:bundle {:optional true} [:maybe [:or bytes? :string]]]+ [:entity_id {:optional true} [:maybe :string]]+ [:name {:optional true} ::slug]+ [:display_name {:optional true} ::display-name]+ [:allowed_hosts {:optional true} ::allowed-hosts]+ [:bundle_path {:optional true} ::bundle-path]+ [:bundle {:optional true} [:maybe ::bundle]][:bundle_hash {:optional true} [:maybe :string]]- [:last_synced_sha {:optional true} [:maybe :string]]- [:last_synced_at {:optional true} [:maybe ms/TemporalInstant]]- [:sync_error {:optional true} [:maybe :string]][:enabled {:optional true} [:maybe :boolean]][:created_at {:optional true} [:maybe ms/TemporalInstant]][:updated_at {:optional true} [:maybe ms/TemporalInstant]]- [:description {:optional true} [:maybe :string]]- [:version {:optional true} ms/PositiveInt]+ [:description {:optional true} ::description]+ [:version {:optional true} ::version][:resource_collection_id {:optional true} [:maybe ms/PositiveInt]][:permission_group_id {:optional true} [:maybe ms/PositiveInt]][:table_ids {:optional true} [:maybe [:sequential ms/PositiveInt]]][:draft {:optional true} [:maybe :boolean]]])++(mr/def ::data-app.insert+ "What an insert of a DataApp accepts: [[::data-app.update]] with its slug, display name, and bundle path required."+ [:merge+ ::data-app.update+ [:map+ [:name ::slug]+ [:display_name ::display-name]+ [:bundle_path ::bundle-path]]])diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/sync.clj b/enterprise/backend/src/metabase_enterprise/data_apps/sync.cljdeleted file mode 100644index 05855ef571b..00000000000--- a/enterprise/backend/src/metabase_enterprise/data_apps/sync.clj+++ /dev/null@@ -1,296 +0,0 @@-(ns metabase-enterprise.data-apps.sync- "Materialize data apps from a synced repository snapshot — discovery, upsert, and- pruning. [[sync-from-snapshot!]] is the entry point remote-sync calls on every- import.-- See `README.md` in this directory for the pipeline, the source-of-truth rules- (what a sync deletes, and what unlinking or switching repos does), and the- failure-isolation guarantees this namespace implements.-- This namespace does no Git access of its own — the snapshot's `read-file` /- `list-dir` come from remote-sync. The cached `bundle` blob is what serving- reads, so a failed sync never takes a working app offline, and the admin- `enabled` toggle is preserved across syncs."- (:require- [clojure.string :as str]- [metabase-enterprise.data-apps.config :as data-app.config]- [metabase-enterprise.data-apps.db :as data-apps.db]- [metabase-enterprise.data-apps.resources :as data-app.resources]- [metabase.permissions.core :as perms]- [metabase.premium-features.core :as premium-features]- [metabase.settings.core :as setting]- [metabase.util.i18n :refer [tru]]- [metabase.util.log :as log]- [toucan2.core :as t2])- (:import- (java.security MessageDigest)- (org.apache.commons.codec.binary Hex)))--(set! *warn-on-reflection* true)--(def ^:const max-bundle-bytes- "Cap on a synced bundle's size (10 MiB)."- (* 10 1024 1024))--(defn- bytes-hash ^String [^bytes b]- (let [^MessageDigest md (MessageDigest/getInstance "SHA-256")]- (Hex/encodeHexString ^bytes (.digest md b))))--(defn- ->bytes ^bytes [^String s]- (.getBytes s "UTF-8"))--(defn repo-url- "The connected remote-sync repository URL, or nil when none is configured. Read- by keyword so this OSS namespace has no compile-time dependency on the- enterprise remote-sync module; returns nil when that module (and its setting)- isn't loaded."- []- (let [url (try (setting/get :remote-sync-url) (catch Throwable _ nil))]- (when-not (str/blank? url)- url)))--(defn- create-draft!- [slug]- (let [app (t2/with-transaction [_conn]- (when-not (data-apps.db/data-app-exists? slug)- (data-apps.db/insert-data-app!- {:name slug- :display_name slug- :bundle_path (format "%s/%s/%s" data-app.config/apps-dir slug data-app.config/config-file-name)- :sync_error (tru "Bundle not synced yet.")- :draft true}))- (data-apps.db/non-blob-data-app-by-slug slug))]- (data-app.resources/ensure-resources! app)))--(defn ensure-draft!- "Create a data app draft when needed and ensure its permission resources.- A later repository import fills the same row with the authoritative manifest- and bundle."- [slug]- (try- (create-draft! slug)- (catch Throwable e- (if (data-apps.db/data-app-exists? slug)- (create-draft! slug)- (throw e)))))--;;; ----------------------------------------------------- Discovery -------------------------------------------------------(defn- discover-app-configs- "Given the snapshot's `list-dir` and `read-file` fns (where `list-dir` returns a- directory's immediate children as repo-root relative paths, and `read-file`- returns file text or nil), iterate the folders under `data_apps/` and return one- entry per folder that is an app — it holds a `data_app.yaml`; a folder without- one is skipped. Each entry is a parsed app `{:slug :display_name :description- :version :bundle :allowed_hosts}` (with `:bundle` the repo-root relative bundle path) or- `{:config-error <message>}`. Parse/read failures are isolated per app so one bad- config can't abort the sync.-- Reads only `data_apps/` and its app folders, never the whole tree. That matters- beyond this fn: `data_apps/` is not a serdes path, so a pull that changes only- data apps builds no ingestable at all (see remote-sync's- `incremental-import-plan`) — discovery here is the only thing that would touch- the tree, and it now stays proportional to the number of apps rather than to- the size of the repo."- [list-dir read-file]- (for [dir (list-dir data-app.config/apps-dir)- :let [config-path (str dir "/" data-app.config/config-file-name)]- ;; a plain file under `data_apps/` lists no children, so it drops out here- :when (some #{config-path} (list-dir dir))]- (try- (if-let [content (read-file config-path)]- (let [{:keys [slug display_name description version path allowed_hosts]}- (data-app.config/parse-app-config (->bytes content) dir)]- {:slug slug- :display_name display_name- :description description- :version version- :bundle (str dir "/" path)- :allowed_hosts allowed_hosts})- {:slug (data-app.config/dir-slug dir), :config-error (tru "Could not read {0}." config-path)})- (catch Throwable e- {:slug (data-app.config/dir-slug dir), :config-error (ex-message e)}))))--;;; ----------------------------------------------------- Materialize -------------------------------------------------------(defn- upsert-by-name!- "Insert or update by slug. Never writes `:enabled`, so the admin toggle (and the- DB default of true for new rows) is preserved across syncs."- [slug row]- (if (data-apps.db/data-app-exists? slug)- (data-apps.db/update-data-app-by-slug! slug row)- (data-apps.db/insert-data-app! (assoc row :name slug))))--(def ^:private synced-metadata-keys- "The manifest-derived columns a sync stores whether or not the bundle loaded."- [:display_name :description :version :allowed_hosts :bundle_path])--(defn- app-metadata-changed?- "Whether the metadata a sync stores whether or not the bundle loaded differs from- the `existing` row. The failure path stores exactly these, so it asks this- directly; the success path adds the bundle to the question."- [existing fields]- (let [normalize (fn [row]- (-> (select-keys row synced-metadata-keys)- (update :allowed_hosts #(vec (or % [])))))]- (not= (normalize existing) (normalize fields))))--(defn- app-content-changed?- "Whether the just-synced content differs from the `existing` row (nil = a new- app). Compares only content-bearing fields — a `last_synced_sha` /- `last_synced_at` bump on an otherwise-identical app is NOT a change, so callers- can count real changes (e.g. for the remote-sync pull summary)."- [existing {:keys [bundle_hash] :as fields}]- (or (nil? existing)- (some? (:sync_error existing))- (not= (:bundle_hash existing) bundle_hash)- (app-metadata-changed? existing fields)))--(defn- mark-config-error!- "Record a `data_app.yaml` parse failure on an app that already has a row, keeping the- row and its cached bundle and setting `sync_error`. An app with no row yet isn't- materialized at all — there's nothing to serve. Returns true when this changed the- app's recorded state, so callers can count it like any other change."- [existing slug message]- (boolean- (when (and existing (not= (:sync_error existing) message))- (data-apps.db/update-data-app-by-slug! slug {:sync_error message})- true)))--(defn- prepare-app- "Read one app's bundle before taking the permissions lock.- A bundle failure preserves the cached bundle and prepares metadata with `sync_error` instead."- [existing {:keys [slug display_name description version bundle sha read-file allowed_hosts]}]- (let [metadata {:display_name display_name- :description description- :version version- :allowed_hosts allowed_hosts- :bundle_path bundle}]- (try- (let [content (read-file bundle)- _ (when-not content- (throw (ex-info (tru "Bundle file \"{0}\" not found in the repository." bundle)- {:status-code 400})))- ^bytes bytes (->bytes content)]- (when (> (alength bytes) max-bundle-bytes)- (throw (ex-info (tru "Bundle for \"{0}\" must be less than {1} MiB."- slug (quot max-bundle-bytes (* 1024 1024)))- {:status-code 413})))- (let [fields (assoc metadata :bundle_hash (bytes-hash bytes))]- {:fields (assoc fields :bundle bytes :last_synced_sha sha- :sync_error nil :draft false)- :changed? (app-content-changed? existing fields)}))- (catch Throwable e- (log/warnf "[data-app] failed to read bundle for app %s: %s" slug (ex-message e))- {:fields (assoc metadata :sync_error (ex-message e))- :changed? (or (nil? existing)- (not= (:sync_error existing) (ex-message e))- (app-metadata-changed? existing metadata))}))))--(defn- record-app-failure!- "Record a failure after the app transaction rolls back. Keep existing content and resource links intact."- [existing slug fields message]- (try- (t2/with-transaction [_conn]- (if existing- (mark-config-error! existing slug message)- (when fields- (upsert-by-name! slug (assoc (select-keys fields synced-metadata-keys)- :sync_error message))- true)))- (catch Throwable e- (log/warnf "[data-app] could not record sync error for app %s: %s" slug (ex-message e))- false)))--(defn- sync-app!- "Commit one app and its permission resources together. Return whether its recorded state changed."- [existing {:keys [slug config-error] :as cfg}]- (let [{:keys [fields changed?]} (when-not config-error (prepare-app existing cfg))]- (try- (perms/with-global-permissions-lock- (t2/with-transaction [_conn]- (data-apps.db/publish-data-app-drafts! [slug])- (if config-error- (mark-config-error! existing slug config-error)- (do- (upsert-by-name! slug fields)- (data-app.resources/ensure-resources! (data-apps.db/non-blob-data-app-by-slug slug))- changed?))))- (catch Throwable e- (log/warnf "[data-app] failed to sync app %s: %s" slug (ex-message e))- (record-app-failure! existing slug fields (ex-message e))))))--(defn- prune-apps!- "Delete all absent apps. Preserve the sync summary if pruning rolls back."- [present-slugs]- (try- {:removed (t2/with-transaction [_conn]- (if (seq present-slugs)- (data-apps.db/delete-data-apps-not-named! present-slugs)- (data-apps.db/delete-all-data-apps!)))}- (catch Throwable e- (log/warnf "[data-app] pruning failed: %s" (ex-message e))- {:removed 0 :pruning-error (ex-message e)})))--(defn import-from-snapshot!- "Materialize data apps from a synced repo `snapshot`:-- {:read-file <(fn [path] -> <file-text-string> | nil)>- :list-dir <(fn [path] -> [<child-path-string> ...])>- :sha <commit-sha-string>}-- Discovers every `data_apps/<dir>/data_app.yaml`, upserts a row per app, and prunes- rows whose directory is gone from the snapshot. Apps are synced independently, so- one failing app cannot roll back the others. Returns- `{:synced <n>, :changed <n>, :removed <n>, :sha <sha>, :config-errors [<msg> ...]}`,- where `:changed` counts apps actually created/updated (a `last_synced_sha` bump on- unchanged content does not count) and `:removed` counts apps dropped for no longer- being in the repo. If pruning fails, `:removed` is zero and `:pruning-error` contains- the error message. Counts for committed app updates are preserved.-- Two apps can't collide on a slug here: a slug *is* an app's directory name (see- the config namespace), a repo can't hold two `data_apps/<slug>` directories, and- discovery takes one config per directory (see [[discover-app-configs]])."- [{:keys [read-file list-dir sha]}]- ;; realize discovery once (it calls read-file/parse per config); `results` is- ;; then walked several times below- (let [results (vec (discover-app-configs list-dir read-file))- good (remove :config-error results)- errors (vec (keep :config-error results))- ;; every app directory present in the repo (parsed or not) — the set of- ;; slugs that should survive this sync- present-slugs (into #{} (map :slug) results)- ;; pre-sync rows, so we can tell a real change from a sha/timestamp bump- existing (into {} (map (juxt :name identity))- (data-apps.db/data-apps-sync-info))- changed (count (filter (fn [{:keys [slug] :as cfg}]- (sync-app! (get existing slug) (assoc cfg :sha sha :read-file read-file)))- results))- {:keys [removed] :as pruning-result} (prune-apps! present-slugs)]- (log/infof "[data-app] synced sha=%s apps=%d changed=%d removed=%d errors=%d"- sha (count good) changed removed (count errors))- (merge {:synced (count good), :changed changed, :sha sha, :config-errors errors}- pruning-result)))--(defn sync-from-snapshot!- "Entry point for the remote-sync import pipeline. Materializes data apps from- the just-imported `snapshot` (see [[import-from-snapshot!]] for its shape).- Never throws, so it can't break the surrounding remote-sync import: a thrown- failure is logged and swallowed, and a malformed `data_app.yaml` is logged and- its app simply doesn't appear. Returns the [[import-from-snapshot!]] result, or- nil if the sync threw.-- A no-op returning nil when the `:data-apps` feature is absent: without- it an instance behaves exactly as if data apps did not exist — no rows,- collections, or permission groups are materialized, and nothing is pruned."- [snapshot]- (when (premium-features/enable-data-apps?)- (try- (let [{:keys [config-errors] :as result} (import-from-snapshot! snapshot)]- (when (seq config-errors)- (log/warnf "[data-app] %d data_app.yaml config(s) skipped: %s"- (count config-errors) (str/join "; " config-errors)))- result)- (catch Throwable e- (log/warnf "[data-app] sync from remote-sync snapshot failed: %s" (ex-message e))- nil))))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.cljindex 26f72f41a61..855fc93a8a1 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj@@ -13,7 +13,7 @@(def ^:private ConditionKey"The column keys used in the `:conditions` / `:cascade-filter` / `:removal-conditions` of a remote-sync modelspec (see `metabase-enterprise.remote-sync.spec`)."- [:enum :exploration_id :built_in_type :active :entity_id :collection_id :archived :archived_at])+ [:enum :exploration_id :built_in_type :active :entity_id :collection_id :archived :archived_at :draft])(def ^:private Conditions"A map of column to value (possibly nil) or Toucan 2 operator-vector value, or nil for none."diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 282b95eeb3e..172cdbd4a86 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -3,7 +3,6 @@[clojure.string :as str][diehard.core :as dh][java-time.api :as t]- [metabase-enterprise.data-apps.sync :as data-apps.sync][metabase-enterprise.remote-sync.db :as remote-sync.db][metabase-enterprise.remote-sync.guards :as guards][metabase-enterprise.remote-sync.merge :as remote-sync.merge]@@ -260,7 +259,7 @@{:model_type model-type:model_id (:id instance):path (or repo-path (:path fspec))- :content_hash (source/content-hash (:content fspec))})+ :content_hash (source/file-spec-hash fspec)})(catch Exception e(log/warnf "Skipping %s %s: failed to serialize for content hash: %s" model-type (:id instance) (ex-message e))nil)))]@@ -308,21 +307,6 @@(and (some? pre-task-branch)(not= pre-task-branch (settings/remote-sync-branch))))-(defn- materialize-data-apps!- "Materialize data apps from the snapshot a content import is landing. Data apps live under `data_apps/` in- the repo (outside the serdes paths), so they ride this import rather than having their own sync. Runs inside- the import's commit transaction, before the version is written: a task's `version` is the sync base (see- [[remote-sync.db/last-synced-task]]) and gates every later pull, so a version that landed without its data- apps would never be re-materialized. Adapts the snapshot to plain reader fns;- `data-apps.sync/sync-from-snapshot!` never throws."- [^SourceSnapshot snapshot]- ;; `read-file` returns file text (a string) or nil; data-apps.sync converts to- ;; bytes on its side, keeping all Java interop out of this namespace.- (data-apps.sync/sync-from-snapshot!- {:read-file (fn [path] (source.p/read-file snapshot path))- :list-dir (fn [path] (source.p/list-dir snapshot path))- :sha (source.p/version snapshot)}))-(defn- import-progress-reporter"A throttled progress reporter (see `make-progress-reporter`) for the import task `task-id`."[task-id]@@ -397,41 +381,31 @@"Models whose change forces a full import on the incremental fast-path: Collection (a rename moves everydescendant's file, a delete cascades to its contents) and the feature models (their presence drives theremote-sync-transforms / library settings, which need whole-snapshot knowledge to toggle correctly)."- #{"Collection" "Transform" "TransformTag" "PythonLibrary" "NativeQuerySnippet"})+ #{"Collection" "DataApp" "Transform" "TransformTag" "PythonLibrary" "NativeQuerySnippet"})-(defn- legal-yaml-path?- "True for a managed-directory `.yaml` entity file — the only changed paths the importer acts on."+(defn- managed-path?+ "True for a file under a managed directory."[^String path]- (and (str/ends-with? path ".yaml")- (not (str/starts-with? path "."))+ (and (not (str/starts-with? path "."))(when-let [i (str/index-of path "/")](contains? serialization/legal-top-level-paths (subs path 0 i)))))+(defn- legal-yaml-path?+ "True for a managed-directory `.yaml` entity file — the only changed paths the importer acts on."+ [^String path]+ (serialization/entity-file-path? path))+(defn- pulled-change-count"Total number of entities applied by a pull, across entity-id- and path-identified models in `imported-data`."[imported-data](transduce (map count) + 0 (concat (vals (:by-entity-id imported-data))(vals (:by-path imported-data)))))-(defn- fold-data-app-changes- "Fold the count of data apps a pull changed — upserted *or* removed — into its- `:outcome`. Data apps live under `data_apps/` — outside serdes — and are- materialized separately (see [[materialize-data-apps!]]), so they're invisible- to [[pulled-change-count]]. Without this, a pull whose only changes are data- apps (including one that only deletes an app) reports `pull-skipped` /- `count 0`. Caller guarantees `da-changed` is positive."- [outcome da-changed]- (case (:kind outcome)- "pull-skipped" {:kind "pulled" :count da-changed :branch (settings/remote-sync-branch)}- "pulled" (update outcome :count (fnil + 0) da-changed)- "merged" (update outcome :pulled (fnil + 0) da-changed)- outcome))-(defn- incremental-import-plan"What an incremental load would touch, or [[incremental-not-possible]] if the change must fall back to a fullimport. On success returns `{:ingestable <ingestable-or-nil> :deleted-rsos <RemoteSyncObject rows>}`. Falls back- when there is no diff, a deleted file can't be mapped back to a tracked entity, or a- structural/feature/non-entity-id model changed."+ when there is no diff, a resource file changed or a non-YAML file was deleted, a deleted file can't be mapped back+ to a tracked entity, or a structural/feature/non-entity-id model changed."[snapshot last-version](let [changed (when last-version (source.p/changed-files snapshot last-version))add-mod (into #{} (filter legal-yaml-path?) (into (:added changed) (:modified changed)))@@ -449,6 +423,10 @@(nil? changed) ;; first import, or diffing not possible (force-push/rebase, non-diffable source):remote-sync/incremental-not-possible+ (or (seq (source/owned-paths snapshot (remove legal-yaml-path? (concat (:added changed) (:modified changed)))))+ (some #(and (managed-path? %) (not (legal-yaml-path? %))) (:deleted changed)))+ :remote-sync/incremental-not-possible+(some nil? deleted-rsos) ;; some deleted file not present in appdb:remote-sync/incremental-not-possible@@ -602,11 +580,7 @@first-import? (nil? last-imported-version);; force-deletion? defaults to force? when a caller doesn't pass it.force-deletion? (if (nil? force-deletion?) force? force-deletion?)- da-result (volatile! nil)- ;; Data apps before the version: the version write locks the task row until commit, which would- ;; block the heartbeat for the whole materialization.finalize! (fn []- (vreset! da-result (materialize-data-apps! snapshot))(remote-sync.task/set-version! task-id snapshot-version))report (import-progress-reporter task-id)path-filters (mapv #(re-pattern (str % "/.*")) serialization/legal-top-level-paths)@@ -711,18 +685,7 @@:outcome {:kind "pulled":count (pulled-change-count imported-data):branch (settings/remote-sync-branch)}}))]- ;; Data apps rode the pull inside `finalize!`, materialized from the real source snapshot (the repo- ;; file tree under `data_apps/`), not the synthetic merged snapshot `load-snapshot!` sees. They're- ;; counted outside serdes, so fold how many they upserted or removed into the outcome — otherwise a- ;; data-app-only pull would report `pull-skipped` / `count 0`. Paths that skipped `finalize!` left- ;; no result: the version did not move, so neither did the apps.- (if (= :success (:status result))- ;; Removals count too: a pull whose only change is deleting an app directory upserts nothing- ;; (`:changed` 0) but still changed what this instance serves, so it must report as a pull, not skipped.- (let [da-changed (+ (:changed @da-result 0) (:removed @da-result 0))]- (cond-> result- (pos? da-changed) (update :outcome fold-data-app-changes da-changed)))- result))+ result)(catch Exception e;; A cancellation isn't a failure: log it and return nil. Otherwise log the error, count it, and;; return a user-friendly :error result.@@ -789,7 +752,7 @@:message "Export blocked: the same content was changed both locally and on the remote branch."})(let [[_ version] (commit-staged! snapshot message(fn [commit]- (source.p/replace-all! commit) ; merged set replaces the managed dirs wholesale+ (source/replace-managed-files! commit snapshot) ; merged set replaces the managed files wholesale(run! #(source.p/stage-upsert! commit %) merged)));; An empty merge means the merged set already matched the remote tip: nothing was pushed, so;; reconcile (and advance) against the tip rather than a non-existent merge commit.@@ -955,6 +918,9 @@(:identity (spec/spec-for-model-type model_type))):remote-sync/incremental-not-possible+ (:resources? (spec/spec-for-model-type model_type))+ :remote-sync/incremental-not-possible+(not (#{"create" "update" "removed" "delete"} status)):remote-sync/incremental-not-possible@@ -1075,11 +1041,11 @@(not (settings/library-is-remote-synced?)) (into ["snippets" "glossary"])))(defn- stage-write [commit opts [row entity]]- (let [path (or (:file_path row) (source/entity->path opts entity))- content (source/entity->content entity)]- (source.p/stage-upsert! commit {:path path :content content})+ (let [path (or (:file_path row) (source/entity->path opts entity))+ fspec (source/entity->file-spec-at path entity)]+ (run! #(source.p/stage-upsert! commit %) (source/file-specs fspec))(when (:id row)- [{:id (:id row) :file_path path :content_hash (source/content-hash content)}])))+ [{:id (:id row) :file_path path :content_hash (source/file-spec-hash fspec)}])))(defn- chunk-stage-writes[commit opts chunk]@@ -1157,7 +1123,7 @@(let [opts (source/storage-context)[synced version] (commit-staged! snapshot message(fn [commit]- (source.p/replace-all! commit) ; replace the managed dirs wholesale+ (source/replace-managed-files! commit snapshot) ; replace the managed files wholesale(let [synced (stage-writes commit opts export-rows(fn [staged](report (+ export-progress-plan-donediff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/merge.cljindex b2c877b0132..a281509bd6d 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj@@ -17,6 +17,7 @@and produces a merged set of file specs plus a list of genuine conflicts (the same entity changeddifferently on both sides)."(:require+ [clojure.string :as str][metabase.models.serialization :as serdes][metabase.util.log :as log][metabase.util.yaml :as yaml]))@@ -41,10 +42,11 @@nil)))(defn- file-key- "Identity key for a `{:path :content}` file spec: the serdes identity when available, otherwise a- path-based fallback so non-serdes files still merge sanely."- [{:keys [path content]}]- (or (entity-identity content)+ "Identity key for a `{:path :content}` file spec: the serdes identity of a YAML file when available, otherwise a+ path-based fallback so non-serdes files, such as resource files, still merge sanely."+ [{:keys [^String path content]}]+ (or (when (str/ends-with? path ".yaml")+ (entity-identity content))[::by-path path]))(defn- index-by-keydiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.cljindex 6fbd471c03d..33de53b88d4 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj@@ -84,28 +84,51 @@(defn entity->content"The serialized YAML string for an extracted `entity`."[entity]- (yaml/generate-string (serialization/serialization-deep-sort entity)+ (yaml/generate-string (serialization/serialization-deep-sort (serialization/without-resources entity)){:dumper-options {:flow-style :block :split-lines false}}))+(defn- resource-specs+ "The `{:path :content}` file specs of `entity`'s resource files, next to its YAML file at `path`."+ [path entity]+ (let [dir (subs path 0 (inc (or (str/last-index-of path "/") -1)))]+ (for [[resource content] (sort-by key (:serdes/resources entity))]+ {:path (str dir resource) :content content})))++(defn entity->file-spec-at+ "Serializes a single extracted entity at the YAML `path` into a `{:path :content :resources}` file spec;+ `:resources` holds the `{:path :content}` specs of its resource files."+ [path entity]+ {:path path+ :content (entity->content entity)+ :resources (vec (resource-specs path entity))})+(defn entity->file-spec- "Serializes a single extracted entity into a `{:path :content}` file spec, using storage context- `opts` (from [[storage-context]])."+ "[[entity->file-spec-at]] the path storage context `opts` (from [[storage-context]]) gives `entity`."[opts entity]- {:path (entity->path opts entity)- :content (entity->content entity)})+ (entity->file-spec-at (entity->path opts entity) entity))++(defn file-specs+ "The `{:path :content}` specs of every file a `file-spec` from [[entity->file-spec]] writes."+ [file-spec]+ (cons (select-keys file-spec [:path :content]) (:resources file-spec)))(defn content-hash"SHA-256 (hex) of a serialized YAML `content` string."[^String content](codecs/bytes->hex (buddy-hash/sha256 content)))+(defn file-spec-hash+ "SHA-256 (hex) of the content of a `file-spec` from [[entity->file-spec]], including its resource files."+ [{:keys [content resources]}]+ (content-hash (apply str content (mapcat (juxt :path :content) resources))))+(defn row->file-info"The repo `:path` and the SHA-256 (hex) `:content-hash` of the serialized YAML for the entity named by `row`({:model_type :model_id}), or nil if it can't be extracted."[row](when-let [entity (first (spec/extract-entities-for-rows [row]))]- (let [{:keys [path content]} (entity->file-spec (storage-context) entity)]- {:path path, :content-hash (content-hash content)})))+ (let [fspec (entity->file-spec (storage-context) entity)]+ {:path (:path fspec), :content-hash (file-spec-hash fspec)})))(defn row->content-hash"SHA-256 (hex) of the serialized YAML for the entity named by `row` ({:model_type :model_id}), or nil if it@@ -132,26 +155,70 @@(remote-sync.task/update-progress! task-id (-> n (/ total) (min 1) (* 0.65) (+ 0.3))))(constantly nil))](into []- (map-indexed (fn [idx entity]- (when (instance? Exception entity)- (throw entity))- (let [spec (entity->file-spec opts entity)]- (report (inc idx))- spec)))+ (comp (map-indexed (fn [idx entity]+ (when (instance? Exception entity)+ (throw entity))+ (let [spec (entity->file-spec opts entity)]+ (report (inc idx))+ (file-specs spec))))+ cat)stream)))+(defn- managed-path? [^String path]+ (when-let [idx (str/index-of path "/")]+ (contains? serialization/legal-top-level-paths (subs path 0 idx))))++(defn- shared-path? [^String path]+ (when-let [idx (str/index-of path "/")]+ (contains? serialization/shared-top-level-paths (subs path 0 idx))))++(defn- parent-dir [^String path]+ (when-let [idx (str/last-index-of path "/")]+ (subs path 0 idx)))++(defn- declared-resources+ "The paths of the resource files that the entity YAML files directly in directory `dir` of `snapshot` declare."+ [snapshot dir]+ (into #{}+ (comp (filter serialization/entity-file-path?)+ (mapcat (fn [path]+ (try+ (let [entity (yaml/parse-string (source.p/read-file snapshot path)+ {:key-fn serialization/parse-key})]+ (map #(str dir "/" %) (serdes/resource-paths entity)))+ (catch Exception _+ nil)))))+ (source.p/list-dir snapshot dir)))++(defn owned-paths+ "The managed paths among `paths` that serialization owns in `snapshot`: entity YAML files, and the resource files+ that a YAML file in one of their ancestor directories declares."+ [snapshot paths]+ (let [resources (memoize #(declared-resources snapshot %))]+ (filterv (fn [path]+ (and (managed-path? path)+ (or (serialization/entity-file-path? path)+ (some #(contains? (resources %) path)+ (take-while some? (iterate parent-dir (parent-dir path)))))))+ paths)))++(defn replace-managed-files!+ "Stage the removal of every file serialization owns in `snapshot` into `commit`, so the files staged next replace+ them wholesale; files in [[serialization/shared-top-level-paths]] that serialization does not own are kept."+ [commit snapshot]+ (source.p/replace-all! commit)+ (run! #(source.p/stage-delete! commit %)+ (owned-paths snapshot (filter shared-path? (source.p/list-files snapshot)))))+(defn- snapshot->specs- "Reads a snapshot's managed-directory files into a sequence of `{:path :content}` specs, matching the+ "Reads the files serialization owns in a snapshot into a sequence of `{:path :content}` specs, matching theshape produced by [[serialize-specs]]. Used to read the merge base and remote-tip trees for merging."[snapshot](into [](keep (fn [path]- (when (contains? serialization/legal-top-level-paths- (when-let [idx (str/index-of path "/")]- (subs path 0 idx)))- (when-let [content (source.p/read-file snapshot path)]- {:path path :content content}))))- (source.p/list-files snapshot)))+ (when-let [content (source.p/read-file snapshot path)]+ {:path path :content content})))+ (owned-paths snapshot (source.p/list-files snapshot))))(defn compute-merge"Runs the entity-identity 3-way merge of local state against the remote tip, without writing. Returns@@ -218,6 +285,6 @@(setting/get :remote-sync-url)(or branch (setting/get :remote-sync-branch))(setting/get :remote-sync-token)- serialization/legal-top-level-paths))+ serialization/replaced-top-level-paths))([](source-from-settings (setting/get :remote-sync-branch))))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.cljindex c643071bf5c..0111ef8fe8d 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj@@ -40,8 +40,7 @@[snapshot](let [errors (atom [])]{:entities (into {} (for [path (source.p/list-files snapshot)- :when (and (not (str/starts-with? path "."))- (str/ends-with? path ".yaml"))+ :when (serialization/entity-file-path? path):let [content (try(source.p/read-file snapshot path)(catch Exception e@@ -147,9 +146,11 @@(ingest-one [_ serdes-path](populate-cache! cache errors-atom #(ingest-all snapshot))- (when-let [target (get @cache (serialization/strip-labels serdes-path))]+ (when-let [{:keys [content ^String path]} (get @cache (serialization/strip-labels serdes-path))](try- (ingest-content (:content target))+ (let [dir (subs path 0 (inc (or (str/last-index-of path "/") -1)))]+ (serialization/read-resources (ingest-content content)+ #(source.p/read-file snapshot (str dir %))))(catch Exception e(throw (ex-info "Unable to ingest file" {:abs-path serdes-path} e))))))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex 26814669d1d..c4735e9f391 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -50,8 +50,9 @@:prefix - Event keyword prefix (e.g., :event/card):types - Vector of event types to handle [:create :update :delete]- :eligibility - Eligibility configuration:- :type - :collection, :published-table, :parent-table, :parent, :setting, or- :library-synced (:parent follows :parent-fk to a :parent-model instance)+ :type - :collection, :published-table, :parent-table, :parent, :setting,+ :library-synced, or :always (:parent follows :parent-fk to a :parent-model+ instance):collection - For :collection type: :remote-synced, :transforms-namespace, :snippets-namespace, or :any:setting - For :setting type: setting keyword to check(Note: :library-synced type uses the library-is-remote-synced? setting to determine eligibility)@@ -76,6 +77,9 @@then scoped to that table's rows in the synced collections (Action):all-on-setting-disable - Optional setting keyword; when this setting's sentinelRSO exists with 'delete' status, remove ALL entities of this type+ - :resources? - Optional. True when the model's YAML files carry resource files (see+ `metabase.models.serialization/resource-paths`), whose changes only a full import or export+ handles.- :export-scope - Export scope for query-export-roots::root-collections - Query root-level remote-synced + namespace collections (Collection):root-only - Query root instances with collection_id = nil (Transform)@@ -364,7 +368,23 @@:removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion:all-on-setting-disable :remote-sync-transforms}:export-scope :all ; query for all instances- :enabled? :remote-sync-transforms}})+ :enabled? :remote-sync-transforms}++ :model/DataApp+ {:model-type "DataApp"+ :model-key :model/DataApp+ :identity :entity-id+ :events {:prefix :event/data-app+ :types [:create :update :delete]}+ :eligibility {:type :always}+ :archived-key nil+ :tracking {:select-fields [:name]+ :field-mappings {:model_name :name}}+ :conditions {:draft false}+ :removal {:statuses #{"removed" "delete"}}+ :resources? true+ :export-scope :all+ :enabled? true}});;; ------------------------------------------------- Helper Functions -------------------------------------------------@@ -742,29 +762,30 @@({:type :category :model :count :names :message}), one per affected model type."[{:keys [by-entity-id]}](let [synced-collection-ids (all-syncable-collection-ids)]- (cond-> []- (seq synced-collection-ids)- (into (for [[model-key spec] (specs-for-deletion)- :when (and (not (get-in spec [:removal :all-on-setting-disable]))- (not= :model/Collection model-key))- :let [model-type (:model-type spec)- imported-ids (get by-entity-id model-type #{})- ;; Same base restriction remove-unsynced! deletes by, plus an anti-join keeping only the- ;; unsynced rows the import would delete. Done in SQL so we never materialize a whole- ;; collection's worth of rows just to count/sample them.- opts (removal-opts spec synced-collection-ids imported-ids)- n (remote-sync.db/unsynced-instance-count model-key model-type opts)- name-col (get-in spec [:tracking :field-mappings :model_name])]- :when (pos? n)]- {:type (keyword (str (u/lower-case-en model-type) "-deletion-conflict"))- :category model-type- :model model-type- :count n- ;; A bounded sample of names for the UI; :count above is the true total.- :names (remote-sync.db/unsynced-instance-names model-key model-type name-col opts- max-conflict-names)- :message (format "Import would delete %d unsynced local %s %s"- n model-type (if (= 1 n) "entity" "entities"))})))))+ (into []+ (for [[model-key spec] (specs-for-deletion)+ :when (and (not (get-in spec [:removal :all-on-setting-disable]))+ (not= :model/Collection model-key)+ (or (seq synced-collection-ids)+ (nil? (get-in spec [:removal :scope-key]))))+ :let [model-type (:model-type spec)+ imported-ids (get by-entity-id model-type #{})+ ;; Same base restriction remove-unsynced! deletes by, plus an anti-join keeping only the+ ;; unsynced rows the import would delete. Done in SQL so we never materialize a whole+ ;; collection's worth of rows just to count/sample them.+ opts (removal-opts spec synced-collection-ids imported-ids)+ n (remote-sync.db/unsynced-instance-count model-key model-type opts)+ name-col (get-in spec [:tracking :field-mappings :model_name])]+ :when (pos? n)]+ {:type (keyword (str (u/lower-case-en model-type) "-deletion-conflict"))+ :category model-type+ :model model-type+ :count n+ ;; A bounded sample of names for the UI; :count above is the true total.+ :names (remote-sync.db/unsynced-instance-names model-key model-type name-col opts+ max-conflict-names)+ :message (format "Import would delete %d unsynced local %s %s"+ n model-type (if (= 1 n) "entity" "entities"))}))))(defn- object-matches-conditions?"True if `object` satisfies every column-value pair in `conditions` (or if no conditions are set).@@ -844,6 +865,10 @@[_spec _object](rs-settings/library-is-remote-synced?))+(defmethod check-eligibility-by-type :always+ [_ _]+ true)+(defmethod check-eligibility-by-type :default[_ _]false)@@ -1238,6 +1263,15 @@(remote-sync.db/ids-where model-key (when archived-key {archived-key false})))nil)))+(defmethod query-export-roots :always+ [{:keys [export-scope model-key model-type] :as spec}]+ (case export-scope+ :all+ (into #{}+ (map (fn [id] [model-type id]))+ (remote-sync.db/ids-where model-key (export-conditions spec)))+ nil))+(defmethod query-export-roots :default [_] nil)(def git-sync-extract-optsdiff --git a/enterprise/backend/src/metabase_enterprise/serialization/core.clj b/enterprise/backend/src/metabase_enterprise/serialization/core.cljindex a1a88a32d9e..ec11ccfe9f9 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/core.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/core.clj@@ -25,13 +25,17 @@make-targets-of-typeextract][metabase-enterprise.serialization.v2.ingest+ entity-file-path?legal-top-level-paths+ replaced-top-level-paths+ shared-top-level-pathsstrip-labelsIngestableingest-yamlingest-listingest-oneingest-errors+ read-resourcesread-timestampsparse-key][metabase-enterprise.serialization.v2.load@@ -42,4 +46,6 @@file-writer][metabase-enterprise.serialization.v2.storage.utilresolve-storage-path- slugify-name])+ resource-files+ slugify-name+ without-resources])diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.cljindex cb35344cf1c..d745db7af81 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj@@ -40,7 +40,10 @@(conj "EmbeddingTheme")(not (:no-custom-viz-plugins opts))- (conj "CustomVizPlugin")))+ (conj "CustomVizPlugin")++ (not (:no-data-apps opts))+ (conj "DataApp")));; OsiAiContext is intentionally NOT in the default export set. It's a top-level model that *depends on* its;; entity, extracted unfiltered, so any export — even an untargeted "full" one, which still omits personal anddiff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.cljindex ce05c58c158..b2fb591cdb5 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj@@ -80,10 +80,54 @@(yaml/from-file {:key-fn parse-key})read-timestamps))+(defn- check-resource-path!+ "Throws unless `path` is a relative path that stays inside its entity's directory."+ [path]+ (when-not (and (string? path)+ (not (str/starts-with? path "/"))+ (not (str/includes? path "\\"))+ (not-any? #{"" "." ".."} (str/split path #"/" -1)))+ (throw (ex-info (format "Invalid resource file path: %s" path) {:path path}))))++(defn read-resources+ "The ingested `entity` with each of its [[serdes/resource-paths]] read by `read-fn` into `:serdes/resources`."+ [entity read-fn]+ (if-let [paths (seq (serdes/resource-paths entity))]+ (assoc entity :serdes/resources (into {}+ (map (fn [path]+ (check-resource-path! path)+ [path (read-fn path)]))+ paths))+ entity))+(def legal-top-level-paths"Known top-level paths for directory with serialization output.We support both \"python-libraries\" and \"python_libraries\" for backwards compatibility. The modern name is \"python_libraries\"."- #{"actions" "channels" "collections" "custom_viz_plugins" "databases" "embedding_themes" "glossary" "metabots" "python_libraries" "python-libraries" "osi_ai_context" "snippets" "transforms"})+ #{"actions" "channels" "collections" "custom_viz_plugins" "data_apps" "databases" "embedding_themes" "glossary" "metabots" "python_libraries" "python-libraries" "osi_ai_context" "snippets" "transforms"})++(def shared-top-level-paths+ "The [[legal-top-level-paths]] whose directories also hold files serialization does not own, such as a data app's+ source code, to the name of the entity files in them."+ {"data_apps" "data_app.yaml"})++(def replaced-top-level-paths+ "The [[legal-top-level-paths]] a full export may replace wholesale."+ (apply disj legal-top-level-paths (keys shared-top-level-paths)))++(defn entity-file-path?+ "Whether the `/`-separated relative `path` names an entity YAML file: a `.yaml` file that isn't a dotfile under one+ of the [[legal-top-level-paths]], with the entity file name of its directory when that is one of the+ [[shared-top-level-paths]]."+ [^String path]+ (let [top (first (str/split path #"/" 2))+ file-name (subs path (inc (or (str/last-index-of path "/") -1)))]+ (boolean (and (str/includes? path "/")+ (str/ends-with? file-name ".yaml")+ (not (str/starts-with? file-name "."))+ (contains? legal-top-level-paths top)+ (if-let [entity-file-name (get shared-top-level-paths top)]+ (= entity-file-name file-name)+ true)))))(defn- path-interner"Returns a function that interns `:serdes/meta` path vectors.@@ -104,14 +148,11 @@(mapv intern-seg hierarchy))))(defn- ingestible-file?- "Whether `file` is a regular `.yaml` file under one of the [[legal-top-level-paths]].- Dotfiles are excluded (editor temp files, see #41567)."+ "Whether `file` is a regular file that [[entity-file-path?]] accepts. Dotfiles are excluded (editor temp files, see+ #41567)."[^File root-dir ^File file]- (boolean (and (.isFile file)- (not (str/starts-with? (.getName file) "."))- (str/ends-with? (.getName file) ".yaml")- (let [rel (.relativize (.toPath root-dir) (.toPath file))]- (-> rel (.subpath 0 1) (.toString) legal-top-level-paths)))))+ (and (.isFile file)+ (entity-file-path? (str/replace (str (.relativize (.toPath root-dir) (.toPath file))) "\\" "/"))))(defn- file-hierarchy!"Parses `file` and returns its `:serdes/meta` abstract path, or nil on parse failure.@@ -164,7 +205,11 @@{:serdes/meta serdes-meta :key kw-id :value (get settings kw-id)})(when-let [file (get @cache (strip-labels serdes-meta))](try- (ingest-file file)+ (read-resources (ingest-file file)+ (fn [path]+ (let [resource (io/file (.getParentFile ^File file) ^String path)]+ (when (.isFile resource)+ (slurp resource)))))(catch Exception e(throw (ex-info "Unable to ingest file" {:file (.getName ^File file):abs-path serdes-meta} e))))))))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.cljindex 44cafccea21..0c12065c267 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj@@ -35,6 +35,7 @@(concat data-modelcontent["CustomVizPlugin"+ "DataApp""EmbeddingTheme""FieldValues""Metabot"@@ -84,7 +85,6 @@"ConnectionImpersonation""ContentTranslation""DashboardBookmark"- "DataApp""DataComplexityScore""DataPermissions""DatabaseRouter"diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.cljindex 92bdff665de..06fb6485ef8 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj@@ -13,11 +13,8 @@(set! *warn-on-reflection* true)-(defn- file ^File [ctx root-dir entity]- (let [resolved (storage.util/resolve-storage-path ctx entity)- dirnames (drop-last resolved)- basename (str (last resolved) ".yaml")]- (apply io/file root-dir (concat dirnames [basename]))))+(defn- yaml-file ^File [root-dir resolved]+ (apply io/file root-dir (concat (drop-last resolved) [(str (last resolved) ".yaml")])))(defn file-writer"Create a filesystem storage backend rooted at `root-dir`."@@ -25,11 +22,16 @@(let [ctx (serdes/storage-base-context)](reify protocols/ExportWriter(store-entity! [_ entity]- (let [f (file ctx root-dir entity)]+ (let [resolved (storage.util/resolve-storage-path ctx entity)+ f (yaml-file root-dir resolved)](log/info "Storing" {:path (serdes/log-path-str (:serdes/meta entity)):file (str (.relativize (Path/of (str root-dir) (make-array String 0))(.toPath f)))})- (spit-yaml! f entity)+ (spit-yaml! f (storage.util/without-resources entity))+ (doseq [[segments content] (storage.util/resource-files resolved entity)+ :let [resource (apply io/file root-dir segments)]]+ (io/make-parents resource)+ (spit resource content))(:serdes/meta entity)))(store-settings! [_ settings]diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.cljindex 368a925bcc7..277218f5331 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj@@ -41,9 +41,11 @@(store-entity! [_ entity](let [resolved (storage.util/resolve-storage-path ctx entity)path (entry-path dirname resolved)- content (.getBytes ^String (yaml-content entity) "UTF-8")]+ content (.getBytes ^String (yaml-content (storage.util/without-resources entity)) "UTF-8")](log/trace "Storing" {:path (serdes/log-path-str (:serdes/meta entity))})(put-entry! tar path content)+ (doseq [[segments ^String resource] (storage.util/resource-files resolved entity)]+ (put-entry! tar (str dirname "/" (str/join "/" segments)) (.getBytes resource "UTF-8")))(:serdes/meta entity)))(store-settings! [_ settings]diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.cljindex 0774eff3303..3bd6adbc7a6 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj@@ -25,15 +25,17 @@(u.str/limit-chars max-label-length))))(defn- resolve-path- "Given a storage path (vector of `{:label ... :key ...}` maps), resolves to a vector of strings+ "Given a storage path (vector of `{:label ... :key ... :style ...}` maps), resolves to a vector of stringswith deduplication per folder."[unique-name-fns path](loop [remaining pathresolved []](if (empty? remaining)resolved- (let [{:keys [label key]} (first remaining)- slug (slugify-name label)+ (let [{:keys [label key style] :or {style :name}} (first remaining)+ slug (case style+ :name (slugify-name label)+ :slug label)gen (or (get @unique-name-fns resolved)(let [g (lib/non-truncating-unique-name-generator)](swap! unique-name-fns assoc resolved g)@@ -47,3 +49,14 @@The last element is the filename (without extension)."[ctx entity](resolve-path (:unique-name-fns ctx) (serdes/storage-path entity ctx)))++(defn without-resources+ "The `entity` as written to its YAML file, without its `:serdes/resources`."+ [entity]+ (dissoc entity :serdes/resources))++(defn resource-files+ "Each of `entity`'s `:serdes/resources` as `[path-segments content]`, next to its YAML file at `resolved`."+ [resolved entity]+ (for [[path content] (sort-by key (:serdes/resources entity))]+ [(into (vec (drop-last resolved)) (str/split path #"/")) content]))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/api_test.cljindex e7621111f0a..6aa15047926 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj@@ -2,12 +2,11 @@(:require[clojure.string :as str][clojure.test :refer :all]+ [metabase-enterprise.data-apps.apps :as data-apps.apps][metabase-enterprise.data-apps.config :as data-app.config][metabase-enterprise.data-apps.query-definition :as query-definition][metabase-enterprise.data-apps.resources :as data-app.resources]- [metabase-enterprise.data-apps.sync :as data-app.sync][metabase-enterprise.data-apps.user-access :as data-app.user-access]- [metabase-enterprise.remote-sync.source :as source][metabase.actions.core :as actions][metabase.api.macros.defendpoint.closed-schemas :as closed-schemas][metabase.lib.core :as lib]@@ -25,36 +24,15 @@(t2/insert! :model/DataApp:name "demo":display_name "Demo"- :bundle_path "data_apps/demo/index.js"- :bundle (.getBytes "BUNDLE" "UTF-8")- :bundle_hash "abc123"))+ :bundle_path "index.js"+ :bundle (.getBytes "BUNDLE" "UTF-8")))-(def ^:private fake-sha "0123456789abcdef0123456789abcdef01234567")--(defn- snapshot- "Build a snapshot (as the remote-sync import passes one) from a path->content- map. `read-file` returns file text (a string) or nil; `list-dir` reuses the- derivation the real non-git snapshots use, so the fake can't drift from it."- [path->content & {:keys [sha] :or {sha fake-sha}}]- {:sha sha- :list-dir (fn [dir] (source/paths->children (keys path->content) dir))- :read-file (fn [p] (get path->content p))})--(defn- app-config- "Render a per-app data_app.yaml from `{:name :path :allowed_hosts}`. No slug: an- app's slug is the name of the directory the config sits in."- [{:keys [name path allowed_hosts]}]- (str (format "name: %s\npath: %s\n" name path)- (when (seq allowed_hosts)- (apply str "allowed_hosts:\n"- (map #(format " - %s\n" %) allowed_hosts)))))--(defn- app-files- "Repo files for one data app under `data_apps/<dir>/`: its data_app.yaml plus a- bundle at `path` with `bundle` content."- [dir {:keys [path bundle] :as cfg}]- {(format "data_apps/%s/data_app.yaml" dir) (app-config cfg)- (format "data_apps/%s/%s" dir path) bundle})+(def ^:private app-request+ {:name "demo"+ :display_name "Demo app"+ :bundle_path "dist/index.js"+ :allowed_hosts ["https://api.example.com"]+ :bundle "DEMOBUNDLE"});;; ---------------------------------------------- Permissions ----------------------------------------------@@ -93,8 +71,7 @@(mt/with-premium-features #{:data-apps}(mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup](create-app!)- (is (nil? (t2/select-one-fn :resource_collection_id :model/DataApp :name "demo"))- "precondition: the app has no resource collection")+ (t2/update! :model/DataApp :name "demo" {:resource_collection_id nil})(testing "an app with no resource collection is not published: neither its metadata nor itsbundle is served — to anyone — and a 409 lets the client show a \"not published\" screen"(doseq [user [:rasta :crowberto]]@@ -380,7 +357,7 @@(deftest superuser-can-store-table-dependencies-test(mt/with-premium-features #{:data-apps}- (mt/with-model-cleanup [:model/DataApp]+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup](create-app!)(let [table-ids [(mt/id :venues) (mt/id :orders)]](is (= (sort table-ids)@@ -396,7 +373,7 @@(deftest non-superuser-cannot-store-table-dependencies-test(mt/with-premium-features #{:data-apps}- (mt/with-model-cleanup [:model/DataApp]+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup](create-app!)(is (= "You don't have permissions to do that."(mt/user-http-request :rasta :put 403 "apps/demo/table-dependencies"@@ -404,7 +381,7 @@(deftest table-dependencies-validates-table-ids-test(mt/with-premium-features #{:data-apps}- (mt/with-model-cleanup [:model/DataApp]+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup](create-app!)(is (= "One or more tables do not exist."(mt/user-http-request :crowberto :put 400 "apps/demo/table-dependencies"@@ -555,7 +532,7 @@(deftest user-permission-warnings-validates-users-test(mt/with-premium-features #{:data-apps :tenants}- (mt/with-model-cleanup [:model/DataApp]+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup](create-app!)(testing "unknown users"(mt/user-http-request :crowberto :post 404 "apps/demo/user-permission-warnings"@@ -570,7 +547,7 @@(deftest non-superuser-cannot-read-user-permission-warnings-test(mt/with-premium-features #{:data-apps}- (mt/with-model-cleanup [:model/DataApp]+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup](create-app!)(is (= "You don't have permissions to do that."(mt/user-http-request :rasta :post 403 "apps/demo/user-permission-warnings"@@ -630,14 +607,14 @@(deftest query-definition-must-use-a-table-source-test(mt/with-premium-features #{:data-apps}- (mt/with-model-cleanup [:model/DataApp]+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup](create-app!)(is (some? (mt/user-http-request :crowberto :post 400 "apps/demo/query"{:stages [{:source {:type "card" :id 1}}]}))))))(deftest query-definition-source-must-be-valid-test(mt/with-premium-features #{:data-apps}- (mt/with-model-cleanup [:model/DataApp]+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup](create-app!)(is (some? (mt/user-http-request :crowberto :post 400 "apps/demo/query"{:stages [{:source {:type 1 :id (mt/id :venues)}}]}))))))@@ -662,7 +639,7 @@(is (= (select-keys first-response [:resource_collection_id :permission_group_id])(select-keys second-response [:resource_collection_id :permission_group_id])))(is (=? {:bundle nil :draft true}- (t2/select-one :model/DataApp :name "draft-app")))))))+ (t2/select-one [:model/DataApp :bundle :draft] :name "draft-app")))))))(deftest non-superuser-cannot-create-a-data-app-draft-test(mt/with-premium-features #{:data-apps}@@ -673,9 +650,9 @@(deftest data-app-draft-must-have-a-valid-slug-test(mt/with-premium-features #{:data-apps}- (mt/with-model-cleanup [:model/DataApp]- (is (= "Data app draft slugs must use lowercase letters, numbers, and dashes."- (mt/user-http-request :crowberto :post 400 "apps/Draft/draft")))+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (is (=? {:errors {:slug #".*lowercase letters.*"}}+ (mt/user-http-request :crowberto :post 400 "apps/Draft/draft")))(is (not (t2/exists? :model/DataApp :name "Draft"))))))(deftest data-app-group-reaches-only-copied-actions-test@@ -686,7 +663,7 @@;; Its own slug: the `Data App: <slug>` group outlives other tests in;; this namespace, so sharing "demo" collides on the group name.(let [{:keys [permission_group_id resource_collection_id]}- (data-app.sync/ensure-draft! "action-perms-app")+ (data-apps.apps/ensure-draft! "action-perms-app")metadata-provider (mt/metadata-provider)venues (lib.metadata/table metadata-provider (mt/id :venues))](perms/add-user-to-group! (mt/user->id :rasta) permission_group_id)@@ -722,8 +699,8 @@(t2/insert! :model/DataApp :name "ready" :display_name "Ready" :bundle_path "data_apps/ready/index.js")(t2/insert! :model/DataApp :name "disabled" :display_name "Disabled" :bundle_path "data_apps/disabled/index.js":enabled false)- (t2/insert! :model/DataApp :name "failed" :display_name "Failed" :bundle_path "data_apps/failed/index.js"- :sync_error "Could not read bundle")+ (t2/insert! :model/DataApp :name "draft" :display_name "Draft" :bundle_path "data_apps/draft/index.js"+ :draft true)(is (=? [{:name "ready" :display_name "Ready"}](mt/user-http-request :rasta :get 200 "apps?available=true"))))))@@ -820,50 +797,75 @@(get-in by-name ["withhosts" :allowed_hosts])))(is (= [] (get-in by-name ["nohosts" :allowed_hosts]))))))))-;;; ----------------------------------------------------- Sync -------------------------------------------------------(deftest import-materializes-apps-test- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [result (data-app.sync/import-from-snapshot!- (snapshot (merge (app-files "sales" {:name "Sales" :path "dist/index.js" :bundle "SALES-BUNDLE"})- (app-files "ops" {:name "Ops" :path "dist/app.js" :bundle "OPS-BUNDLE"}))))]- (is (=? {:synced 2} result))- (is (= #{"sales" "ops"} (t2/select-fn-set :name :model/DataApp)))- (let [sales (t2/select-one :model/DataApp :name "sales")]- (is (= "SALES-BUNDLE" (String. ^bytes (:bundle sales) "UTF-8")))- (is (= "Sales" (:display_name sales)))- (is (= "data_apps/sales/dist/index.js" (:bundle_path sales)))- (is (true? (:enabled sales)))- (is (= fake-sha (:last_synced_sha sales)))- (is (nil? (:sync_error sales)))))))+;;; ------------------------------------------------- Create & update --------------------------------------------------(deftest import-stores-allowed-hosts-test- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (testing "allowed_hosts from data_app.yaml are persisted on the row"- (data-app.sync/import-from-snapshot!- (snapshot (app-files "sales" {:name "Sales" :path "dist/index.js" :bundle "B"- :allowed_hosts ["https://api.example.com" "https://*.acme.com"]})))- (is (= ["https://api.example.com" "https://*.acme.com"]- (:allowed_hosts (t2/select-one :model/DataApp :name "sales")))))- (testing "re-syncing without allowed_hosts clears them to an empty list"- (data-app.sync/import-from-snapshot!- (snapshot (app-files "sales" {:name "Sales" :path "dist/index.js" :bundle "B"})))- (is (= [] (:allowed_hosts (t2/select-one :model/DataApp :name "sales")))))))+(deftest create-endpoint-test+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{:data-apps}+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (testing "a non-superuser cannot create an app"+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :post 403 "apps" app-request))))+ (testing "a superuser creates an app with its manifest fields and bundle"+ (is (=? {:name "demo"+ :display_name "Demo app"+ :bundle_path "dist/index.js"+ :allowed_hosts ["https://api.example.com"]+ :version 1+ :enabled true+ :draft false+ :bundle_hash string?+ :resource_collection_id pos-int?+ :permission_group_id pos-int?}+ (mt/user-http-request :crowberto :post 200 "apps"+ (assoc app-request :bundle_path "./dist/index.js"))))+ (is (str/includes? (str (mt/user-real-request :crowberto :get 200 "apps/demo/bundle"))+ "DEMOBUNDLE")))+ (testing "a taken slug is refused"+ (is (= "A data app with this slug already exists."+ (mt/user-http-request :crowberto :post 409 "apps" app-request))))+ (testing "invalid manifest fields are refused"+ (doseq [bad [{:name "Not A Slug"}+ {:name "repo-status"}+ {:bundle_path "../escape.js"}+ {:allowed_hosts ["ftp://example.com"]}+ {:version 0}]]+ (mt/user-http-request :crowberto :post 400 "apps" (merge app-request {:name "other"} bad))+ (is (not (t2/exists? :model/DataApp :name "other")) (str "should refuse: " (pr-str bad)))))))))++(deftest create-endpoint-fills-a-draft-test+ (mt/with-premium-features #{:data-apps}+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (let [draft (mt/user-http-request :crowberto :post 200 "apps/demo/draft")]+ (is (=? {:id (:id draft)+ :draft false+ :resource_collection_id (:resource_collection_id draft)}+ (mt/user-http-request :crowberto :post 200 "apps" app-request)))))))-(deftest import-prunes-apps-absent-from-snapshot-test- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (data-app.sync/import-from-snapshot!- (snapshot (merge (app-files "keep" {:name "Keep" :path "index.js" :bundle "KEEP"})- (app-files "gone" {:name "Gone" :path "index.js" :bundle "GONE"}))))- (is (= #{"keep" "gone"} (t2/select-fn-set :name :model/DataApp)))- ;; The connected repo is the source of truth: an app whose directory is gone- ;; from a later snapshot is pruned. (An admin can also remove one explicitly- ;; via DELETE /api/apps/:slug.)- (is (=? {:removed 1}- (data-app.sync/import-from-snapshot!- (snapshot (app-files "keep" {:name "Keep" :path "index.js" :bundle "KEEP"})))))- (is (= #{"keep"} (t2/select-fn-set :name :model/DataApp))- "the app absent from the later snapshot is pruned")))+(deftest update-endpoint-test+ (mt/test-helpers-set-global-values!+ (mt/with-premium-features #{:data-apps}+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (let [{:keys [bundle_hash]} (mt/user-http-request :crowberto :post 200 "apps" app-request)]+ (testing "a non-superuser cannot update an app"+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :put 403 "apps/demo" {:display_name "Mine"}))))+ (testing "a superuser updates manifest fields and the bundle"+ (let [updated (mt/user-http-request :crowberto :put 200 "apps/demo"+ {:display_name "Renamed"+ :description " What it does "+ :allowed_hosts []+ :bundle "NEWBUNDLE"})]+ (is (=? {:display_name "Renamed" :description "What it does" :allowed_hosts []}+ updated))+ (is (not= bundle_hash (:bundle_hash updated)))+ (is (str/includes? (str (mt/user-real-request :crowberto :get 200 "apps/demo/bundle"))+ "NEWBUNDLE"))))+ (testing "invalid manifest fields are refused"+ (mt/user-http-request :crowberto :put 400 "apps/demo" {:bundle_path "/abs.js"})+ (is (= "dist/index.js" (t2/select-one-fn :bundle_path :model/DataApp :name "demo"))))+ (testing "updating a missing app 404s"+ (mt/user-http-request :crowberto :put 404 "apps/missing" {:enabled false})))))))(deftest delete-endpoint-test(mt/test-helpers-set-global-values!@@ -891,83 +893,16 @@(testing "removing a non-existent app 404s"(mt/user-http-request :crowberto :delete 404 "apps/missing"))))))))-(deftest import-preserves-enabled-across-syncs-test- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (data-app.sync/import-from-snapshot!- (snapshot (app-files "a" {:name "A" :path "index.js" :bundle "V1"})))- (t2/update! :model/DataApp :name "a" {:enabled false})- ;; a new bundle must not flip the admin's enabled toggle back on- (data-app.sync/import-from-snapshot!- (snapshot (app-files "a" {:name "A" :path "index.js" :bundle "V2"})))- (let [a (t2/select-one :model/DataApp :name "a")]- (is (false? (:enabled a)) "the disabled toggle is preserved")- (is (= "V2" (String. ^bytes (:bundle a) "UTF-8")) "the bundle is still updated"))))--(deftest import-per-app-error-test- (testing "a missing bundle file fails just that app, not the whole import"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (data-app.sync/import-from-snapshot!- (snapshot (merge (app-files "good" {:name "Good" :path "index.js" :bundle "GOOD"})- ;; "bad" declares a path that doesn't exist- {"data_apps/bad/data_app.yaml" (app-config {:name "Bad" :path "missing.js"})})))- (is (= #{"good" "bad"} (t2/select-fn-set :name :model/DataApp)))- (let [good (t2/select-one :model/DataApp :name "good")- bad (t2/select-one :model/DataApp :name "bad")]- (is (= "GOOD" (String. ^bytes (:bundle good) "UTF-8")))- (is (nil? (:sync_error good)))- (is (nil? (:bundle bad)))- (is (str/includes? (:sync_error bad) "missing.js"))))))--(deftest import-serves-each-app-from-its-directory-test- (testing "the directory an app lives in is the slug it's served at — two apps can't collide on one"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (data-app.sync/import-from-snapshot!- (snapshot (merge (app-files "one" {:name "One" :path "a.js" :bundle "A"})- (app-files "two" {:name "Two" :path "b.js" :bundle "B"}))))- (is (= #{"one" "two"} (t2/select-fn-set :name :model/DataApp))))))--(deftest import-isolates-bad-config-test- (testing "a malformed data_app.yaml is isolated: sibling apps in the same repo still sync and are not pruned, the bad one is reported"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (data-app.sync/import-from-snapshot!- (snapshot (app-files "existing" {:name "Existing" :path "i.js" :bundle "E"})))- ;; The repo still holds "existing" and adds "good", alongside a broken "bad".- ;; The broken config must not abort the others, nor prune its siblings.- (let [result (data-app.sync/import-from-snapshot!- (snapshot (merge (app-files "existing" {:name "Existing" :path "i.js" :bundle "E"})- (app-files "good" {:name "Good" :path "i.js" :bundle "GOOD"})- {"data_apps/bad/data_app.yaml" "name: [unterminated"})))]- (is (=? {:synced 2 :removed 0} result))- (is (= 1 (count (:config-errors result))))- (is (= #{"existing" "good"} (t2/select-fn-set :name :model/DataApp))- "the bad config neither aborts nor prunes its sibling apps, and doesn't materialize itself")))))--(deftest sync-from-snapshot!-never-throws-test- (mt/with-premium-features #{:data-apps}- (testing "a malformed data_app.yaml is isolated into :config-errors; the app just doesn't appear, the sync doesn't throw"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [result (data-app.sync/sync-from-snapshot!- (snapshot {"data_apps/x/data_app.yaml" "name: [unterminated"}))]- (is (seq (:config-errors result)))- (is (empty? (t2/select-fn-set :name :model/DataApp))))))- (testing "a clean sync materializes the app with no config errors"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [result (data-app.sync/sync-from-snapshot!- (snapshot (app-files "a" {:name "A" :path "index.js" :bundle "A"})))]- (is (empty? (:config-errors result)))- (is (= #{"a"} (t2/select-fn-set :name :model/DataApp))))))))-;;; ----------------------------------------------------- API -----------------------------------------------------(deftest list-and-bundle-endpoints-test(mt/test-helpers-set-global-values!(mt/with-premium-features #{:data-apps}(mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (data-app.sync/import-from-snapshot!- (snapshot (app-files "demo" {:name "Demo app" :path "dist/index.js" :bundle "DEMOBUNDLE"})))- (testing "GET / lists the synced apps"+ (mt/user-http-request :crowberto :post 200 "apps" app-request)+ (testing "GET / lists the apps"(is (=? [{:name "demo" :display_name "Demo app"- :bundle_path "data_apps/demo/dist/index.js" :enabled true}]+ :bundle_path "dist/index.js" :enabled true}](mt/user-http-request :crowberto :get 200 "apps"))))(testing "GET /:slug/bundle serves the cached bytes"(is (str/includes?@@ -977,11 +912,11 @@(deftest repo-status-endpoint-test(mt/with-premium-features #{:data-apps}(testing "reports no repository when none is connected"- (mt/with-dynamic-fn-redefs [data-app.sync/repo-url (constantly nil)]+ (mt/with-dynamic-fn-redefs [data-apps.apps/repo-url (constantly nil)](is (=? {:configured false :url nil}(mt/user-http-request :crowberto :get 200 "apps/repo-status")))))(testing "reports the connected repository URL"- (mt/with-dynamic-fn-redefs [data-app.sync/repo-url (constantly "https://github.com/metabase/stats-remote-sync")]+ (mt/with-dynamic-fn-redefs [data-apps.apps/repo-url (constantly "https://github.com/metabase/stats-remote-sync")](is (=? {:configured true :url "https://github.com/metabase/stats-remote-sync"}(mt/user-http-request :crowberto :get 200 "apps/repo-status")))))))@@ -989,8 +924,7 @@(mt/test-helpers-set-global-values!(mt/with-premium-features #{:data-apps}(mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (data-app.sync/import-from-snapshot!- (snapshot (app-files "demo" {:name "Demo" :path "index.js" :bundle "BUNDLE"})))+ (mt/user-http-request :crowberto :post 200 "apps" app-request)(testing "PUT /:slug can disable an app"(is (=? {:name "demo" :enabled false}(mt/user-http-request :crowberto :put 200 "apps/demo" {:enabled false}))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/apps_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/apps_test.cljnew file mode 100644index 00000000000..f8e0c8d89ce--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/data_apps/apps_test.clj@@ -0,0 +1,37 @@+(ns metabase-enterprise.data-apps.apps-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.data-apps.apps :as data-apps.apps]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^:private app-row+ {:name "sales"+ :display_name "Sales"+ :bundle_path "dist/index.js"+ :bundle (.getBytes "BUNDLE" "UTF-8")})++(deftest concurrent-draft-creation-is-safe-test+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (let [drafts (doall (repeatedly 2 #(future (data-apps.apps/ensure-draft! "draft-app"))))]+ (doseq [draft drafts]+ @draft)+ (is (= 1 (t2/count :model/DataApp :name "draft-app"))))))++(deftest create-app-fills-a-draft-with-the-same-slug-test+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (let [{:keys [resource_collection_id]} (data-apps.apps/ensure-draft! "sales")+ draft-id (t2/select-one-pk :model/DataApp :name "sales")]+ (is (= draft-id (data-apps.apps/create-app! app-row)))+ (is (=? {:display_name "Sales"+ :draft false+ :resource_collection_id resource_collection_id}+ (t2/select-one :model/DataApp draft-id))))))++(deftest create-app-refuses-a-taken-slug-test+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (data-apps.apps/create-app! app-row)+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"already exists"+ (data-apps.apps/create-app! app-row)))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/config_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/config_test.cljdeleted file mode 100644index 44c8e8f31b3..00000000000--- a/enterprise/backend/test/metabase_enterprise/data_apps/config_test.clj+++ /dev/null@@ -1,146 +0,0 @@-(ns metabase-enterprise.data-apps.config-test- (:require- [clojure.java.io :as io]- [clojure.test :refer :all]- [metabase-enterprise.data-apps.config :as data-app.config]))--(set! *warn-on-reflection* true)--(defn- ->bytes ^bytes [^String s]- (.getBytes s "UTF-8"))--(defn- parse- "Parse a `data_app.yaml` as if it sat in `dir` (default `data_apps/sales`, so the- parsed slug is `sales` — the slug is the directory's name, never a config key)."- ([s] (parse s "data_apps/sales"))- ([s dir] (data-app.config/parse-app-config (->bytes s) dir)))--(deftest parse-valid-config-test- (is (= {:slug "sales" :display_name "Sales dashboard" :description nil :version 1- :path "dist/index.js" :allowed_hosts []}- (parse "name: Sales dashboard-path: ./dist/index.js"))))--(deftest parse-version-test- (testing "absent means 1: every app predates the field"- (is (= 1 (:version (parse "name: X\npath: dist/index.js")))))- (testing "a whole number is carried through"- (is (= 3 (:version (parse "name: X\nversion: 3\npath: dist/index.js")))))- (testing "anything but a positive whole number is rejected, not coerced or compared as-is"- (doseq [bad ["0" "-1" "1.5" "'1'" "one" "[1]" "1.0.0"]]- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"version.*positive whole number"- (parse (str "name: X\nversion: " bad "\npath: dist/index.js")))- (str "should reject: " (pr-str bad))))))--(deftest template-manifest-declares-the-supported-version-test- (testing "the scaffolding template stamps the version this Metabase serves, so a new app is never born outdated"- (is (= data-app.config/supported-app-version- (:version (parse (slurp "skills/metabase-data-app-setup/template/data_app.yaml")))))))--(deftest migration-upgrade-guides-cover-every-version-test- (testing "the migrate skill ships one guide per upgrade up to the supported version, so an app at any older version has a path"- (let [upgrades (->> (.listFiles (io/file "skills/metabase-data-app-migrate/references/upgrades"))- (keep #(re-matches #"v(\d+)-to-v(\d+)\.md" (.getName ^java.io.File %)))- (map (fn [[_ from to]] [(parse-long from) (parse-long to)]))- sort)]- (is (= (map (fn [n] [n (inc n)]) (range 1 data-app.config/supported-app-version))- upgrades)))))--(deftest outdated-test- (testing "an app below the supported version is outdated; one at it is not"- (with-redefs [data-app.config/supported-app-version 2]- (is (data-app.config/outdated? {:version 1}))- (is (not (data-app.config/outdated? {:version 2}))))))--(deftest parse-description-test- (testing "an optional one-liner is trimmed and carried through"- (is (= "Pipeline health by region"- (:description (parse "name: X\ndescription: ' Pipeline health by region '\npath: dist/index.js")))))- (testing "absent → nil (the field is optional)"- (is (nil? (:description (parse "name: X\npath: dist/index.js")))))- (testing "blank → nil, so an unfilled placeholder doesn't become an empty description"- (doseq [blank ["" "' '"]]- (is (nil? (:description (parse (str "name: X\ndescription: " blank "\npath: dist/index.js"))))- (str "should be nil for: " (pr-str blank)))))- (testing "a YAML block scalar spanning several lines is folded onto one"- (is (= "First line second line"- (:description (parse "name: X\ndescription: |\n First line\n second line\npath: dist/index.js")))))- (testing "a description at the length cap is accepted"- (is (= 255- (count (:description (parse (str "name: X\ndescription: " (apply str (repeat 255 "x"))- "\npath: dist/index.js")))))))- (testing "one over the cap is rejected — a paragraph here would be stored in full and returned on every list request"- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"description.*255 characters or fewer"- (parse (str "name: X\ndescription: " (apply str (repeat 256 "x"))- "\npath: dist/index.js")))))- (testing "the cap applies to the folded value, not the raw YAML"- (is (= 255- (count (:description (parse (str "name: X\ndescription: >\n " (apply str (repeat 255 "x"))- "\npath: dist/index.js")))))))- (testing "a non-string description (a YAML list or mapping) is rejected, not str-ified into the row"- (doseq [bad ["description: [foo, bar]"- "description:\n key: value"]]- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"description.*must be a one-line string"- (parse (str "name: X\n" bad "\npath: dist/index.js")))- (str "should reject: " (pr-str bad))))))--(deftest slug-comes-from-the-directory-test- (testing "the app's slug is the name of the directory it lives in"- (is (= "inventory" (:slug (parse "name: X\npath: dist/index.js" "data_apps/inventory")))))- (testing "a directory name that isn't a usable slug is rejected — the app would have no URL to be served at"- (doseq [bad ["Sales" "my_app" "sales app" "-sales" "sales-"]]- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"lowercase letters, numbers, and dashes"- (parse "name: X\npath: dist/index.js" (str "data_apps/" bad)))- (str "should reject the directory: " bad)))))--(deftest parse-allowed-hosts-test- (testing "valid entries are lowercased, trailing-slash-stripped, and de-duplicated"- (is (= ["https://api.example.com" "https://*.internal.acme.com"]- (:allowed_hosts- (parse "name: X\npath: dist/index.js\nallowed_hosts:\n - https://API.example.com/\n - https://*.internal.acme.com\n - https://api.example.com")))))- (testing "absent → empty vector"- (is (= [] (:allowed_hosts (parse "name: X\npath: dist/index.js")))))- (testing "a non-list value is rejected"- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"allowed_hosts.*must be a list"- (parse "name: X\npath: dist/index.js\nallowed_hosts: https://api.example.com"))))- (testing "invalid entries (bare *, path, non-http scheme, no scheme) are rejected"- (doseq [bad ["*" "https://example.com/path" "ftp://example.com" "example.com"]]- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"not a valid allowed_hosts entry"- (parse (str "name: X\npath: dist/index.js\nallowed_hosts:\n - \"" bad "\"")))- (str "should reject: " bad)))))--(deftest parse-strips-leading-dot-slash-test- (is (= "dist/app.js"- (:path (parse "name: X\npath: ./dist/app.js")))))--(deftest unknown-fields-are-ignored-test- (testing "unknown keys don't fail the parse — including a stray `slug`, which the directory name overrides"- (is (= {:slug "sales" :display_name "X" :description nil :version 1 :path "dist/index.js" :allowed_hosts []}- (parse "name: X\nslug: elsewhere\nfuture_option: 1\npath: dist/index.js")))))--(deftest parse-errors-test- (testing "missing name"- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"name"- (parse "path: dist/index.js"))))- (testing "missing path"- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"path"- (parse "name: X"))))- (testing "malformed yaml"- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Could not parse"- (parse "name: [unterminated"))))- (testing "path traversal is rejected"- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"must not contain"- (parse "name: X\npath: ../other/dist/index.js")))- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"must not contain"- (parse "name: X\npath: dist/../../escape.js"))))- (testing "a directory whose name collides with an API sub-route is rejected"- (doseq [slug ["repo-status" "sandbox-host"]]- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"reserved slug"- (parse "name: X\npath: dist/index.js" (str "data_apps/" slug)))))))--(deftest parse-errors-carry-400-test- (try- (parse "name: X")- (is false "should have thrown")- (catch clojure.lang.ExceptionInfo e- (is (= 400 (:status-code (ex-data e)))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/models/data_app_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/models/data_app_test.cljindex 516c1b090d5..5a14a67a634 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/models/data_app_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/models/data_app_test.clj@@ -1,7 +1,6 @@(ns metabase-enterprise.data-apps.models.data-app-test- "Unit coverage for the DataApp model: the blob-excluding reads, JSON that never- leaks the bundle bytes, the `allowed_hosts` NULL→[] read coercion, and the- superuser-only permission gating."+ "Unit coverage for the DataApp model: the write hooks' invariants, the blob-excluding reads, JSON that never+ leaks the bundle bytes, the `allowed_hosts` NULL→[] read coercion, and the superuser-only permission gating."(:require[clojure.test :refer :all][metabase-enterprise.data-apps.db :as data-apps.db]@@ -19,32 +18,72 @@:model/DataApp(merge {:name "m":display_name "M"- :bundle_path "data_apps/m/index.js"- :bundle (.getBytes "BUNDLEBYTES" "UTF-8")- :bundle_hash "hash"}+ :bundle_path "index.js"+ :bundle (.getBytes "BUNDLEBYTES" "UTF-8")}extra)))+(deftest bundle-hash-follows-the-bundle-test+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (let [id (insert-app! :bundle_hash "stale")+ hash #(t2/select-one-fn :bundle_hash :model/DataApp id)+ v1 (hash)]+ (is (= 64 (count v1)) "the hash is computed from the bundle, not taken from the row")+ (t2/update! :model/DataApp id {:display_name "Renamed"})+ (is (= v1 (hash)) "a change that leaves the bundle alone keeps its hash")+ (t2/update! :model/DataApp id {:display_name "Renamed"})+ (is (= v1 (hash)) "an update that changes nothing is a no-op")+ (t2/update! :model/DataApp id {:bundle (.getBytes "OTHER" "UTF-8")})+ (is (not= v1 (hash)))+ (t2/update! :model/DataApp id {:bundle nil})+ (is (nil? (hash))))))++(deftest insert-creates-the-resources-the-app-owns-test+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (let [{:keys [resource_collection_id permission_group_id]} (t2/select-one :model/DataApp (insert-app!))]+ (is (t2/exists? :model/Collection :id resource_collection_id))+ (is (t2/exists? :model/PermissionsGroup :id permission_group_id :is_data_app_group true)))))++(deftest writes-are-normalized-and-validated-against-the-schema-test+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (let [id (insert-app! :display_name " M " :bundle_path "./dist/index.js"+ :allowed_hosts ["https://API.example.com/"])]+ (is (=? {:display_name "M" :bundle_path "dist/index.js" :version 1 :allowed_hosts ["https://api.example.com"]}+ (t2/select-one :model/DataApp id)))+ (testing "an update with an invalid column is refused"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"does not match schema"+ (t2/update! :model/DataApp id {:bundle_path "../escape.js"})))+ (is (= "dist/index.js" (t2/select-one-fn :bundle_path :model/DataApp id)))))+ (testing "an insert without a required manifest column is refused"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"does not match schema"+ (t2/insert! :model/DataApp {:name "x" :bundle_path "i.js"}))))+ (testing "an insert with an invalid slug is refused"+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"does not match schema"+ (insert-app! :name "Not A Slug"))))))+(deftest to-json-never-includes-the-bundle-bytes-test- (mt/with-model-cleanup [:model/DataApp]+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup](insert-app!)- (let [app (t2/select-one :model/DataApp :name "m")+ (let [app (t2/select-one [:model/DataApp :bundle :display_name] :name "m")decoded (json/decode (json/encode app))]- (is (contains? app :bundle) "the selected instance still carries the raw bundle")+ (is (contains? app :bundle) "the instance selected with its bundle carries the raw bytes")(is (not (contains? decoded "bundle"))"but the JSON representation omits it")(is (= "M" (get decoded "display_name"))))))-(deftest non-blob-selects-exclude-the-bundle-test- (mt/with-model-cleanup [:model/DataApp]+(deftest default-selects-exclude-the-bundle-test+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup](insert-app! :allowed_hosts ["https://api.example.com"])- (testing "non-blob-data-app-by-slug returns metadata without the bundle blob"- (let [app (data-apps.db/non-blob-data-app-by-slug "m")]+ (is (not (contains? (t2/select-one :model/DataApp :name "m") :bundle)))+ (is (= "BUNDLEBYTES" (String. ^bytes (data-apps.db/data-app-bundle (t2/select-one-pk :model/DataApp :name "m"))+ "UTF-8")))+ (testing "data-app-by-slug returns metadata without the bundle blob"+ (let [app (data-apps.db/data-app-by-slug "m")](is (not (contains? app :bundle)))(is (= "M" (:display_name app)))(is (= ["https://api.example.com"] (:allowed_hosts app)))))))(deftest allowed-hosts-reads-as-a-vector-never-nil-test- (mt/with-model-cleanup [:model/DataApp]+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup](testing "a row stored with NULL allowed_hosts reads back as []"(insert-app!)(is (= [] (:allowed_hosts (t2/select-one :model/DataApp :name "m")))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/schema_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/schema_test.cljnew file mode 100644index 00000000000..0a5678d4cef--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/data_apps/schema_test.clj@@ -0,0 +1,111 @@+(ns metabase-enterprise.data-apps.schema-test+ (:require+ [clojure.java.io :as io]+ [clojure.test :refer :all]+ [metabase-enterprise.data-apps.config :as data-app.config]+ [metabase-enterprise.data-apps.schema :as data-apps.schema]+ [metabase.lib.core :as lib]+ [metabase.util.malli.registry :as mr]+ [metabase.util.yaml :as yaml]))++(set! *warn-on-reflection* true)++(defn- normalize [schema x]+ (lib/normalize schema x))++(defn- valid? [schema x]+ (mr/validate schema (normalize schema x)))++(deftest data-app-update-normalizes-each-column-it-is-given-test+ (is (= {:name "sales" :display_name "Sales dashboard" :description nil :version 1+ :bundle_path "dist/index.js" :allowed_hosts []}+ (normalize ::data-apps.schema/data-app.update+ {:name "sales" :display_name " Sales dashboard " :description " " :version nil+ :bundle_path "./dist/index.js" :allowed_hosts nil})))+ (is (= {:enabled false} (normalize ::data-apps.schema/data-app.update {:enabled false}))))++(deftest data-app-insert-requires-the-manifest-columns-test+ (is (valid? ::data-apps.schema/data-app.insert {:name "a" :display_name "A" :bundle_path "i.js"}))+ (doseq [k [:name :display_name :bundle_path]]+ (is (not (valid? ::data-apps.schema/data-app.insert+ (dissoc {:name "a" :display_name "A" :bundle_path "i.js"} k)))+ (str "should require " k))))++(deftest version-test+ (testing "nil means 1: every app predates the field"+ (is (= 1 (normalize ::data-apps.schema/version nil))))+ (testing "anything but a positive whole number is rejected, not coerced"+ (doseq [bad [0 -1 1.5 "1" "one" [1]]]+ (is (not (valid? ::data-apps.schema/version bad)) (str "should reject: " (pr-str bad))))))++(deftest template-manifest-declares-the-supported-version-test+ (testing "the scaffolding template stamps the version this Metabase serves, so a new app is never born outdated"+ (is (= data-app.config/supported-app-version+ (:version (yaml/parse-string (slurp "skills/metabase-data-app-setup/template/data_app.yaml")))))))++(deftest migration-upgrade-guides-cover-every-version-test+ (testing "the migrate skill ships one guide per upgrade up to the supported version, so an app at any older version has a path"+ (let [upgrades (->> (.listFiles (io/file "skills/metabase-data-app-migrate/references/upgrades"))+ (keep #(re-matches #"v(\d+)-to-v(\d+)\.md" (.getName ^java.io.File %)))+ (map (fn [[_ from to]] [(parse-long from) (parse-long to)]))+ sort)]+ (is (= (map (fn [n] [n (inc n)]) (range 1 data-app.config/supported-app-version))+ upgrades)))))++(deftest outdated-test+ (testing "an app below the supported version is outdated; one at it is not"+ (with-redefs [data-app.config/supported-app-version 2]+ (is (data-app.config/outdated? {:version 1}))+ (is (not (data-app.config/outdated? {:version 2}))))))++(deftest description-test+ (testing "an optional one-liner is trimmed, and a multi-line value folded onto one line"+ (is (= "Pipeline health by region" (normalize ::data-apps.schema/description " Pipeline health by region ")))+ (is (= "First line second line" (normalize ::data-apps.schema/description "First line\n second line\n"))))+ (testing "blank → nil, so an unfilled placeholder doesn't become an empty description"+ (doseq [blank [nil "" " "]]+ (is (nil? (normalize ::data-apps.schema/description blank)) (str "should be nil for: " (pr-str blank)))))+ (testing "the length cap applies to the folded value — a paragraph here would be returned on every list request"+ (is (valid? ::data-apps.schema/description (apply str (repeat 255 "x"))))+ (is (not (valid? ::data-apps.schema/description (apply str (repeat 256 "x"))))))+ (testing "a non-string description (a YAML list or mapping) is rejected, not str-ified into the row"+ (doseq [bad [["foo" "bar"] {:key "value"}]]+ (is (not (valid? ::data-apps.schema/description bad)) (str "should reject: " (pr-str bad))))))++(deftest slug-test+ (testing "a slug is used verbatim"+ (is (= "inventory-2" (normalize ::data-apps.schema/slug "inventory-2"))))+ (testing "a slug that can't appear in a URL as-is, or collides with an API sub-route, is rejected"+ (doseq [bad [nil "" "Sales" "my_app" "sales app" "-sales" "sales-" "sales\n" "repo-status" "sandbox-host"+ (apply str (repeat 101 "a"))]]+ (is (not (valid? ::data-apps.schema/slug bad)) (str "should reject: " (pr-str bad))))))++(deftest display-name-test+ (is (= "X" (normalize ::data-apps.schema/display-name " X ")))+ (doseq [bad [nil "" " "]]+ (is (not (valid? ::data-apps.schema/display-name bad)) (str "should reject: " (pr-str bad)))))++(deftest allowed-hosts-test+ (testing "valid entries are lowercased, trailing-slash-stripped, and de-duplicated"+ (is (= ["https://api.example.com" "https://*.internal.acme.com"]+ (normalize ::data-apps.schema/allowed-hosts+ ["https://API.example.com/" "https://*.internal.acme.com" "https://api.example.com"]))))+ (testing "nil → empty vector"+ (is (= [] (normalize ::data-apps.schema/allowed-hosts nil))))+ (testing "a non-list value is rejected"+ (is (not (valid? ::data-apps.schema/allowed-hosts "https://api.example.com"))))+ (testing "invalid entries (bare *, path, non-http scheme, no scheme) are rejected"+ (doseq [bad ["*" "https://example.com/path" "ftp://example.com" "example.com" "https://a.com\nx"]]+ (is (not (valid? ::data-apps.schema/allowed-hosts [bad])) (str "should reject: " bad)))))++(deftest bundle-path-test+ (testing "a leading ./ is dropped"+ (is (= "dist/app.js" (normalize ::data-apps.schema/bundle-path "./dist/app.js"))))+ (testing "a missing path, or one leaving the app's directory, is rejected"+ (doseq [bad [nil "" " " "../other/dist/index.js" "dist/../../escape.js" "/etc/passwd" "dist//index.js"+ "..\\..\\escape.js" "data_app.yaml"]]+ (is (not (valid? ::data-apps.schema/bundle-path bad)) (str "should reject: " (pr-str bad))))))++(deftest bundle-test+ (is (valid? ::data-apps.schema/bundle (byte-array 3)))+ (is (not (valid? ::data-apps.schema/bundle (byte-array (inc data-apps.schema/max-bundle-bytes))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.cljnew file mode 100644index 00000000000..9394bd9cdf4--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.clj@@ -0,0 +1,179 @@+(ns metabase-enterprise.data-apps.serialization-test+ (:require+ [clojure.java.io :as io]+ [clojure.test :refer :all]+ [metabase-enterprise.data-apps.apps :as data-apps.apps]+ [metabase-enterprise.serialization.core :as serialization]+ [metabase-enterprise.serialization.test-util :as ts]+ [metabase-enterprise.serialization.v2.extract :as extract]+ [metabase.models.serialization :as serdes]+ [metabase.test :as mt]+ [metabase.util.yaml :as yaml]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(defn- insert-app! [& {:as extra}]+ (t2/insert-returning-instance! :model/DataApp+ (merge {:name "sales-ops"+ :display_name "Sales Ops"+ :bundle_path "dist/index.js"+ :bundle (.getBytes "console.log(1)" "UTF-8")+ :allowed_hosts ["https://api.example.com"]}+ extra)))++(defn- export! [dir]+ (serialization/store! (serdes/extract-all "DataApp" {}) (serialization/file-writer dir)))++(defn- import! [dir]+ (serialization/load-metabase! (serialization/ingest-yaml dir)))++(defn- bundle-text [app]+ (String. ^bytes (t2/select-one-fn :bundle [:model/DataApp :bundle] :id (:id app)) "UTF-8"))++(defn- write-app-files!+ "Write a serialized data app with `yaml` fields and the `files` next to it under `data_apps/<dir>/`."+ [dump-dir dir yaml-fields files]+ (let [app-dir (io/file dump-dir "data_apps" dir)]+ (.mkdirs app-dir)+ (spit (io/file app-dir "data_app.yaml") (yaml/generate-string yaml-fields))+ (doseq [[path content] files+ :let [f (io/file app-dir ^String path)]]+ (io/make-parents f)+ (spit f content))))++(defn- app-yaml [entity-id slug & {:as extra}]+ (merge {:serdes/meta [{:model "DataApp" :id entity-id :label slug}]+ :entity_id entity-id+ :slug slug+ :name "Sales"+ :path "dist/index.js"}+ extra))++(deftest export-writes-the-manifest-and-the-bundle-file-test+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (ts/with-random-dump-dir [dump-dir "data-app-export-"]+ (let [app (insert-app! :description "Pipeline health")]+ (export! dump-dir)+ (testing "the manifest is a serdes YAML in the app's directory, keyed like a hand-written data_app.yaml"+ (is (= {:serdes/meta [{:model "DataApp" :id (:entity_id app) :label "sales_ops"}]+ :entity_id (:entity_id app)+ :slug "sales-ops"+ :name "Sales Ops"+ :description "Pipeline health"+ :path "dist/index.js"+ :allowed_hosts ["https://api.example.com"]}+ (dissoc (yaml/from-file (io/file dump-dir "data_apps" "sales-ops" "data_app.yaml"))+ :created_at))))+ (testing "the bundle is a plain file at its path next to the manifest"+ (is (= "console.log(1)"+ (slurp (io/file dump-dir "data_apps" "sales-ops" "dist" "index.js")))))))))++(deftest drafts-are-not-exported-test+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (data-apps.apps/ensure-draft! "draft-app")+ (is (empty? (into [] (serdes/extract-all "DataApp" {}))))))++(deftest round-trip-test+ (mt/with-premium-features #{:data-apps}+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (ts/with-random-dump-dir [dump-dir "data-app-round-trip-"]+ (let [app (insert-app!)]+ (export! dump-dir)+ (t2/delete! :model/DataApp (:id app))+ (import! dump-dir)+ (let [imported (t2/select-one :model/DataApp :entity_id (:entity_id app))]+ (is (=? {:name "sales-ops"+ :display_name "Sales Ops"+ :bundle_path "dist/index.js"+ :bundle_hash (:bundle_hash app)+ :allowed_hosts ["https://api.example.com"]+ :draft false}+ imported))+ (is (= "console.log(1)" (bundle-text imported)))+ (testing "the import creates the collection and permission group the app owns"+ (is (t2/exists? :model/Collection :id (:resource_collection_id imported)))+ (is (t2/exists? :model/PermissionsGroup :id (:permission_group_id imported))))))))))++(deftest import-updates-in-place-and-keeps-local-state-test+ (mt/with-premium-features #{:data-apps}+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (ts/with-random-dump-dir [dump-dir "data-app-update-"]+ (let [app (insert-app! :enabled false)]+ (write-app-files! dump-dir "sales-ops" (app-yaml (:entity_id app) "sales-ops" :name "Renamed"+ :path "./dist/index.js")+ {"dist/index.js" "console.log(2)"})+ (import! dump-dir)+ (let [updated (t2/select-one :model/DataApp :id (:id app))]+ (is (=? {:display_name "Renamed"+ :enabled false+ :resource_collection_id (:resource_collection_id app)+ :permission_group_id (:permission_group_id app)}+ updated))+ (is (= "console.log(2)" (bundle-text updated)))+ (is (not= (:bundle_hash app) (:bundle_hash updated)))))))))++(deftest import-takes-over-a-draft-with-the-same-slug-test+ (mt/with-premium-features #{:data-apps}+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (ts/with-random-dump-dir [dump-dir "data-app-draft-"]+ (let [draft (data-apps.apps/ensure-draft! "draft-app")+ id (t2/select-one-pk :model/DataApp :name "draft-app")]+ (write-app-files! dump-dir "draft_app" (app-yaml "Ld3cXiYs9n8HP3q3FvC7R" "draft-app")+ {"dist/index.js" "BUNDLE"})+ (import! dump-dir)+ (is (=? {:id id+ :entity_id "Ld3cXiYs9n8HP3q3FvC7R"+ :draft false+ :resource_collection_id (:resource_collection_id draft)}+ (t2/select-one :model/DataApp :name "draft-app"))))))))++(deftest import-rejects-invalid-apps-test+ (mt/with-premium-features #{:data-apps}+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (testing "a missing bundle file"+ (ts/with-random-dump-dir [dump-dir "data-app-missing-"]+ (write-app-files! dump-dir "x" (app-yaml "pZrj7PDuz3vSWYYi0QFhd" "x") {})+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Failed to load"+ (import! dump-dir)))))+ (testing "a manifest field that fails validation"+ (ts/with-random-dump-dir [dump-dir "data-app-invalid-"]+ (write-app-files! dump-dir "x" (app-yaml "pZrj7PDuz3vSWYYi0QFhd" "Not A Slug")+ {"dist/index.js" "BUNDLE"})+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Failed to load"+ (import! dump-dir)))))+ (testing "a bundle path leaving the app's directory is never read"+ (ts/with-random-dump-dir [dump-dir "data-app-traversal-"]+ (write-app-files! dump-dir "x" (app-yaml "pZrj7PDuz3vSWYYi0QFhd" "x" :path "../../secret.js") {})+ (is (= "Invalid resource file path: ../../secret.js"+ (try+ (serialization/ingest-one (serialization/ingest-yaml dump-dir)+ [{:model "DataApp" :id "pZrj7PDuz3vSWYYi0QFhd"}])+ nil+ (catch clojure.lang.ExceptionInfo e+ (ex-message (ex-cause e))))))))+ (is (not (t2/exists? :model/DataApp))))))++(deftest import-clears-an-omitted-description-test+ (mt/with-premium-features #{:data-apps}+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (ts/with-random-dump-dir [dump-dir "data-app-description-"]+ (let [app (insert-app! :description "Pipeline health")]+ (write-app-files! dump-dir "sales-ops" (app-yaml (:entity_id app) "sales-ops") {"dist/index.js" "B"})+ (import! dump-dir)+ (is (nil? (t2/select-one-fn :description :model/DataApp :id (:id app)))))))))++(deftest import-does-not-take-over-an-app-that-is-not-a-draft-test+ (mt/with-premium-features #{:data-apps}+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (ts/with-random-dump-dir [dump-dir "data-app-takeover-"]+ (let [app (insert-app!)]+ (write-app-files! dump-dir "sales-ops" (app-yaml "Ld3cXiYs9n8HP3q3FvC7R" "sales-ops") {"dist/index.js" "B"})+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Failed to load" (import! dump-dir)))+ (is (=? {:entity_id (:entity_id app)} (t2/select-one :model/DataApp :id (:id app)))))))))++(deftest export-includes-data-apps-test+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ (let [app (insert-app!)]+ (is (some #(= [{:model "DataApp" :id (:entity_id app)}] (map (fn [m] (dissoc m :label)) (:serdes/meta %)))+ (into [] (extract/extract {:no-collections true :no-data-model true :no-settings true})))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/sync_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/sync_test.cljdeleted file mode 100644index 97b73fa380b..00000000000--- a/enterprise/backend/test/metabase_enterprise/data_apps/sync_test.clj+++ /dev/null@@ -1,386 +0,0 @@-(ns metabase-enterprise.data-apps.sync-test- "Unit coverage for the materialization edge cases not exercised through the API- suite: the `:changed` accounting (content vs. sha/timestamp bumps) and the- oversized-bundle guard (rejected with a sync_error, previously cached bundle- retained)."- (:require- [clojure.string :as str]- [clojure.test :refer :all]- [metabase-enterprise.data-apps.db :as data-apps.db]- [metabase-enterprise.data-apps.resources :as data-app.resources]- [metabase-enterprise.data-apps.sync :as data-app.sync]- [metabase-enterprise.remote-sync.source :as source]- [metabase.permissions.core :as perms]- [metabase.test :as mt]- [toucan2.core :as t2]))--(set! *warn-on-reflection* true)--(def ^:private fake-sha "0123456789abcdef0123456789abcdef01234567")--(defn- snapshot- [path->content & {:keys [sha] :or {sha fake-sha}}]- {:sha sha- :list-dir (fn [dir] (source/paths->children (keys path->content) dir))- :read-file (fn [p] (get path->content p))})--(defn- app-files- "The repo files for one app in `data_apps/<dir>`. `dir` is the app's slug — the- config declares no slug, it is the directory's name."- [dir {:keys [name path bundle description version]}]- {(format "data_apps/%s/data_app.yaml" dir)- (str (format "name: %s\npath: %s\n" name path)- (when description (format "description: %s\n" description))- (when version (format "version: %s\n" version)))- (format "data_apps/%s/%s" dir path) bundle})--(deftest sync-from-snapshot-is-gated-by-the-data-apps-feature-test- (testing "the remote-sync entry point does nothing without the :data-apps feature —- an instance then behaves exactly as if data apps did not exist"- (let [files (app-files "a" {:name "A" :path "index.js" :bundle "V1"})]- (mt/with-premium-features #{}- (mt/with-model-cleanup [:model/DataApp]- (is (nil? (data-app.sync/sync-from-snapshot! (snapshot files)))- "returns nil rather than a sync result")- (is (not (t2/exists? :model/DataApp :name "a"))- "materializes no data app")))- (mt/with-premium-features #{:data-apps}- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (is (=? {:synced 1 :changed 1}- (data-app.sync/sync-from-snapshot! (snapshot files)))- "with the feature it materializes the app as usual")- (is (t2/exists? :model/DataApp :name "a")))))))--(deftest resource-provisioning-failures-are-isolated-test- (mt/with-model-cleanup [:model/DataApp]- (let [files (merge (app-files "broken" {:name "Broken" :path "index.js" :bundle "BROKEN"})- (app-files "working" {:name "Working" :path "index.js" :bundle "WORKING"}))]- (mt/with-dynamic-fn-redefs [data-app.resources/ensure-resources!- (fn [app]- (when (= "broken" (:name app))- (throw (ex-info "Resource provisioning failed." {}))))]- (is (=? {:synced 2 :changed 2}- (data-app.sync/import-from-snapshot! (snapshot files))))- (is (= "Resource provisioning failed."- (t2/select-one-fn :sync_error :model/DataApp :name "broken")))- (is (nil? (t2/select-one-fn :sync_error :model/DataApp :name "working")))))))--(deftest sync-creates-stable-permission-resources-test- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [database-id (mt/id)- files (app-files "sales" {:name "Sales" :path "index.js" :bundle "V1"})]- (data-app.sync/import-from-snapshot! (snapshot files))- (let [{:keys [resource_collection_id permission_group_id]}- (t2/select-one :model/DataApp :name "sales")]- (testing "the first sync creates a dedicated group and collection"- (is (pos-int? resource_collection_id))- (is (pos-int? permission_group_id))- (is (= "Data App: sales"- (t2/select-one-fn :name :model/Collection :id resource_collection_id)))- (is (= "Data App: sales"- (t2/select-one-fn :name :model/PermissionsGroup :id permission_group_id)))- (is (t2/exists? :model/Permissions- :group_id permission_group_id- :object (perms/collection-read-path resource_collection_id)))- (is (not (t2/exists? :model/Permissions- :group_id permission_group_id- :object (perms/collection-readwrite-path resource_collection_id))))- (let [permissions (t2/select-fn-vec :perm_value :model/DataPermissions- :group_id permission_group_id- :db_id database-id- :perm_type :perms/create-queries)]- (is (seq permissions))- (is (every? #(= :no %) permissions))))- (testing "later syncs reuse the same resources"- (data-app.sync/import-from-snapshot! (snapshot files))- (is (=? {:resource_collection_id resource_collection_id- :permission_group_id permission_group_id}- (t2/select-one :model/DataApp :name "sales"))))- (testing "removing the app deletes its resources"- (data-app.sync/import-from-snapshot! (snapshot {}))- (is (not (t2/exists? :model/Collection :id resource_collection_id)))- (is (not (t2/exists? :model/PermissionsGroup :id permission_group_id))))))))--(deftest sync-restores-deleted-collection-and-permission-group-test- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [files (app-files "sales" {:name "Sales" :path "index.js" :bundle "V1"})]- (data-app.sync/import-from-snapshot! (snapshot files))- (let [before (select-keys (t2/select-one :model/DataApp :name "sales")- [:resource_collection_id :permission_group_id])]- (t2/delete! :model/Collection :id (:resource_collection_id before))- (t2/delete! :model/PermissionsGroup :id (:permission_group_id before))- (data-app.sync/import-from-snapshot! (snapshot files))- ; after syncing the app again, the data app collection and- ; permission group should be re-created- (let [after (select-keys (t2/select-one :model/DataApp :name "sales") (keys before))]- (is (every? pos-int? (vals after)))- (is (t2/exists? :model/Collection :id (:resource_collection_id after)))- (is (t2/exists? :model/PermissionsGroup :id (:permission_group_id after)))- (is (not= before after)))))))--(deftest changed-count-tracks-content-not-sha-bumps-test- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [files (app-files "a" {:name "A" :path "index.js" :bundle "V1"})]- (testing "the first sync counts the new app"- (is (=? {:synced 1 :changed 1}- (data-app.sync/import-from-snapshot! (snapshot files)))))- (testing "re-syncing identical content at a new sha is not a change"- (is (=? {:synced 1 :changed 0}- (data-app.sync/import-from-snapshot!- (snapshot files :sha "ffffffffffffffffffffffffffffffffffffffff")))))- (testing "a metadata-only change counts"- (is (=? {:changed 1}- (data-app.sync/import-from-snapshot!- (snapshot (app-files "a" {:name "A renamed" :path "index.js" :bundle "V1"}))))))- (testing "a bundle content change counts"- (is (=? {:changed 1}- (data-app.sync/import-from-snapshot!- (snapshot (app-files "a" {:name "A renamed" :path "index.js" :bundle "V2"})))))))))--(deftest description-is-optional-and-tracked-like-other-metadata-test- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [sync-app (fn [& {:as app}]- (data-app.sync/import-from-snapshot!- (snapshot (app-files "a" (merge {:name "A" :path "index.js" :bundle "V1"} app)))))]- (testing "an app that declares no description syncs with a nil one"- (sync-app)- (is (nil? (t2/select-one-fn :description :model/DataApp :name "a"))))- (testing "adding one counts as a change and is materialized"- (is (=? {:changed 1} (sync-app :description "What this app does")))- (is (= "What this app does" (t2/select-one-fn :description :model/DataApp :name "a"))))- (testing "re-syncing the same description is not a change"- (is (=? {:changed 0} (sync-app :description "What this app does"))))- (testing "dropping it from the config clears the column"- (is (=? {:changed 1} (sync-app)))- (is (nil? (t2/select-one-fn :description :model/DataApp :name "a")))))))--(deftest version-is-optional-and-tracked-like-other-metadata-test- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [sync-app (fn [& {:as app}]- (data-app.sync/import-from-snapshot!- (snapshot (app-files "a" (merge {:name "A" :path "index.js" :bundle "V1"} app)))))]- (testing "an app that declares no version syncs as version 1"- (sync-app)- (is (= 1 (t2/select-one-fn :version :model/DataApp :name "a"))))- (testing "declaring one counts as a change and is materialized"- (is (=? {:changed 1} (sync-app :version 2)))- (is (= 2 (t2/select-one-fn :version :model/DataApp :name "a"))))- (testing "re-syncing the same version is not a change"- (is (=? {:changed 0} (sync-app :version 2))))- (testing "dropping it from the config falls back to version 1"- (is (=? {:changed 1} (sync-app)))- (is (= 1 (t2/select-one-fn :version :model/DataApp :name "a"))))- (testing "an invalid version is a config error that keeps the app's last good state"- (is (re-find #"\"version\" must be a positive whole number"- (first (:config-errors (sync-app :version "1.0.0")))))- (is (= 1 (t2/select-one-fn :version :model/DataApp :name "a")))))))--(deftest metadata-edits-count-while-an-app-keeps-failing-test- (testing "an app whose bundle is missing still stores metadata edits, so they count as changes"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [sync-app (fn [& {:as app}]- (let [files (app-files "a" (merge {:name "A" :path "index.js" :bundle "unused"} app))]- (data-app.sync/import-from-snapshot!- (snapshot (dissoc files "data_apps/a/index.js")))))]- (is (=? {:changed 1} (sync-app :description "First")) "the first failure is a change")- (is (some? (t2/select-one-fn :sync_error :model/DataApp :name "a")))- (testing "re-syncing the same failing app unchanged is not a change"- (is (=? {:changed 0} (sync-app :description "First"))))- (testing "editing the description is a change even though the bundle still fails"- (is (=? {:changed 1} (sync-app :description "Second")))- (is (= "Second" (t2/select-one-fn :description :model/DataApp :name "a"))- "the edit is stored, which is why the pull cannot report it as a no-op"))))))--(deftest switching-repos-prunes-old-apps-overrides-shared-adds-new-test- (testing "syncing a different repo: drop apps only the old repo had, override shared slugs, add new ones"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- ;; Repo A: Foo + Bar- (data-app.sync/import-from-snapshot!- (snapshot (merge (app-files "foo" {:name "Foo" :path "index.js" :bundle "FOO"})- (app-files "bar" {:name "Bar A" :path "index.js" :bundle "BAR-A"}))))- (is (= #{"foo" "bar"} (t2/select-fn-set :name :model/DataApp)))- ;; Repo B (Bar + Baz): the repo is the source of truth, so Foo (absent from B)- ;; is pruned, Bar is overridden by slug, and Baz is added.- (is (=? {:synced 2 :removed 1}- (data-app.sync/import-from-snapshot!- (snapshot (merge (app-files "bar" {:name "Bar B" :path "index.js" :bundle "BAR-B"})- (app-files "baz" {:name "Baz" :path "index.js" :bundle "BAZ"}))))))- (is (= #{"bar" "baz"} (t2/select-fn-set :name :model/DataApp))- "Foo (only in repo A) is dropped; Baz (from repo B) is added")- (let [bar (t2/select-one :model/DataApp :name "bar")]- (is (= "Bar B" (:display_name bar))- "Bar is overridden in place by repo B (shared slug)")- (is (= "BAR-B" (String. ^bytes (:bundle bar) "UTF-8"))- "Bar's cached bundle is repo B's")))))--(deftest an-empty-repo-prunes-all-apps-test- (testing "syncing a repo with no data_apps/ removes every app (the repo has none)"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (data-app.sync/import-from-snapshot!- (snapshot (app-files "solo" {:name "Solo" :path "index.js" :bundle "S"})))- (is (= #{"solo"} (t2/select-fn-set :name :model/DataApp)))- (is (=? {:synced 0 :removed 1}- (data-app.sync/import-from-snapshot! (snapshot {}))))- (is (empty? (t2/select-fn-set :name :model/DataApp))))))--(deftest remote-sync-preserves-unpublished-data-app-drafts-test- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (data-app.sync/ensure-draft! "draft-app")- (is (=? {:removed 0}- (data-app.sync/import-from-snapshot! (snapshot {}))))- (is (true? (t2/select-one-fn :draft :model/DataApp :name "draft-app")))- (data-app.sync/import-from-snapshot!- (snapshot (app-files "draft-app" {:name "Draft" :path "index.js" :bundle "BUNDLE"})))- (is (false? (t2/select-one-fn :draft :model/DataApp :name "draft-app")))- (is (=? {:removed 1}- (data-app.sync/import-from-snapshot! (snapshot {}))))- (is (not (t2/exists? :model/DataApp :name "draft-app")))))--(deftest repository-claims-a-draft-before-a-successful-import-test- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (data-app.sync/ensure-draft! "draft-app")- (data-app.sync/import-from-snapshot!- (snapshot {"data_apps/draft-app/data_app.yaml" "name: Draft\n"}))- (is (false? (t2/select-one-fn :draft :model/DataApp :name "draft-app")))- (is (=? {:removed 1}- (data-app.sync/import-from-snapshot! (snapshot {}))))- (is (not (t2/exists? :model/DataApp :name "draft-app")))))--(deftest concurrent-draft-creation-is-safe-test- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [drafts (doall (repeatedly 2 #(future (data-app.sync/ensure-draft! "draft-app"))))]- (doseq [draft drafts]- @draft)- (is (= 1 (t2/count :model/DataApp :name "draft-app"))))))--(deftest remote-sync-prunes-never-successful-repository-apps-test- (mt/with-model-cleanup [:model/DataApp]- (data-app.sync/import-from-snapshot!- (snapshot {"data_apps/broken/data_app.yaml" "name: Broken\npath: missing.js\n"}))- (is (false? (t2/select-one-fn :draft :model/DataApp :name "broken")))- (is (=? {:removed 1}- (data-app.sync/import-from-snapshot! (snapshot {}))))- (is (not (t2/exists? :model/DataApp :name "broken")))))--(deftest a-broken-config-does-not-prune-the-existing-app-test- (testing "a directory that still exists but whose data_app.yaml is now broken keeps the app (as a sync_error), it is not pruned"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (data-app.sync/import-from-snapshot!- (snapshot (app-files "app" {:name "App" :path "index.js" :bundle "GOOD"})))- (is (= "GOOD" (String. ^bytes (:bundle (t2/select-one :model/DataApp :name "app")) "UTF-8")))- ;; Same directory, but its config no longer parses. The dir is still present,- ;; so the app is kept — not treated as a removal — and its cached bundle stays.- (let [result (data-app.sync/import-from-snapshot!- (snapshot {"data_apps/app/data_app.yaml" "name: App\n" ; missing required "path"- "data_apps/app/index.js" "GOOD"}))]- (is (=? {:removed 0} result) "the app is not pruned")- (is (= 1 (count (:config-errors result))))- (is (=? {:changed 1} result)- "marking the app failed counts as a change, so the pull isn't reported as a no-op"))- (let [app (t2/select-one :model/DataApp :name "app")]- (is (some? app) "the app survives a transiently broken config")- (is (= "GOOD" (String. ^bytes (:bundle app) "UTF-8"))- "its last-good cached bundle is retained")- (is (str/includes? (:sync_error app) "path")- "the parse failure is recorded on the row, so the UI shows it as failed rather than freshly synced")))))--(deftest a-broken-config-for-a-brand-new-app-materializes-nothing-test- (testing "an app whose config never parsed has no row to mark — it simply isn't materialized"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [result (data-app.sync/import-from-snapshot!- (snapshot {"data_apps/newbie/data_app.yaml" "name: Newbie\n" ; missing required "path"- "data_apps/newbie/index.js" "X"}))]- (is (= 1 (count (:config-errors result))))- (is (=? {:synced 0 :changed 0 :removed 0} result))- (is (empty? (t2/select-fn-set :name :model/DataApp)))))))--(defn- oversized-bundle ^String []- (.repeat "a" (int (inc data-app.sync/max-bundle-bytes))))--(deftest oversized-bundle-is-rejected-test- (testing "a bundle over the size cap is rejected with a sync_error, no bundle cached"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (data-app.sync/import-from-snapshot!- (snapshot (app-files "big" {:name "Big" :path "index.js"- :bundle (oversized-bundle)})))- (let [app (t2/select-one :model/DataApp :name "big")]- (is (some? app) "the app still appears in the list")- (is (nil? (:bundle app)) "no oversized bundle was cached")- (is (str/includes? (:sync_error app) "MiB"))))))--(deftest oversized-resync-keeps-the-previous-bundle-test- (testing "an oversized re-sync sets sync_error but keeps the last good bundle"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (data-app.sync/import-from-snapshot!- (snapshot (app-files "app" {:name "App" :path "index.js" :bundle "GOOD"})))- (data-app.sync/import-from-snapshot!- (snapshot (app-files "app" {:name "App" :path "index.js"- :bundle (oversized-bundle)})))- (let [app (t2/select-one :model/DataApp :name "app")]- (is (= "GOOD" (String. ^bytes (:bundle app) "UTF-8"))- "the previously cached bundle is retained")- (is (str/includes? (:sync_error app) "MiB"))))))--(deftest a-directory-without-a-config-is-not-an-app-test- (testing "a data_apps/<dir> that ships a bundle but no data_app.yaml is not discovered — no app, no error"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [result (data-app.sync/import-from-snapshot!- (snapshot {"data_apps/orphan/index.js" "BUNDLE"}))]- (is (=? {:synced 0 :changed 0 :config-errors []} result))- (is (empty? (t2/select-fn-set :name :model/DataApp)))))))--(deftest an-unreadable-config-is-a-config-error-test- (testing "a data_app.yaml the snapshot lists but can't read is isolated as a config-error, not a crash"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- ;; the config is listed in the tree, but reading its blob yields nothing- (let [result (data-app.sync/import-from-snapshot!- (snapshot {"data_apps/ghost/data_app.yaml" nil}))]- (is (= 1 (count (:config-errors result))))- (is (str/includes? (first (:config-errors result)) "data_apps/ghost/data_app.yaml"))- (is (empty? (t2/select-fn-set :name :model/DataApp)))))))--(deftest resource-failure-rolls-back-only-that-app-test- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [files (merge (app-files "broken" {:name "Broken" :path "index.js" :bundle "OLD"})- (app-files "working" {:name "Working" :path "index.js" :bundle "OLD"}))- ensure-resources! (mt/original-fn #'data-app.resources/ensure-resources!)]- (data-app.sync/import-from-snapshot! (snapshot files))- (let [collection-id (t2/select-one-fn :resource_collection_id :model/DataApp :name "broken")- old-name (t2/select-one-fn :name :model/Collection :id collection-id)]- ; inject an error into `ensure-resources!` for the "broken" app- (mt/with-dynamic-fn-redefs [data-app.resources/ensure-resources!- (fn [app]- (ensure-resources! app)- (when (= "broken" (:name app))- (t2/update! :model/Collection :id collection-id {:name "Partial update"})- (throw (ex-info "Resource provisioning failed." {}))))]- (data-app.sync/import-from-snapshot!- (snapshot (assoc files "data_apps/broken/index.js" "NEW" "data_apps/working/index.js" "NEW"))))- ; the "broken" app bundle with sync error should be rolled back to "OLD"- (is (= "OLD" (String. ^bytes (t2/select-one-fn :bundle :model/DataApp :name "broken") "UTF-8")))- ; the collection name should be rolled back to the old name- (is (= old-name (t2/select-one-fn :name :model/Collection :id collection-id)))- ; the thrown exception in ensure-resources! should be reported in the sync error- (is (= "Resource provisioning failed." (t2/select-one-fn :sync_error :model/DataApp :name "broken")))- ; the "working" app without sync errors should not be rolled back to "OLD"- (is (= "NEW" (String. ^bytes (t2/select-one-fn :bundle :model/DataApp :name "working") "UTF-8")))))))--(deftest pruning-failure-does-not-roll-back-synced-apps-test- (mt/with-premium-features #{:data-apps}- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [working-files (app-files "working" {:name "Working" :path "index.js" :bundle "OLD"})- files (merge working-files (app-files "removed" {:name "Removed" :path "index.js" :bundle "OLD"}))- prune! (mt/original-fn #'data-apps.db/delete-data-apps-not-named!)]- (data-app.sync/import-from-snapshot! (snapshot files))- ; inject an error into deleting data apps- (mt/with-dynamic-fn-redefs [data-apps.db/delete-data-apps-not-named!- (fn [slugs]- (prune! slugs)- (throw (ex-info "Pruning failed." {})))]- ; sync should report the error from pruning step- (is (=? {:synced 1 :changed 1 :removed 0 :pruning-error "Pruning failed."}- (data-app.sync/sync-from-snapshot!- (snapshot (assoc working-files "data_apps/working/index.js" "NEW"))))))- ; a pruning failure should not roll back the updated app's bundle- (is (= "NEW" (String. ^bytes (t2/select-one-fn :bundle :model/DataApp :name "working") "UTF-8")))- (is (t2/exists? :model/DataApp :name "removed"))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.cljdeleted file mode 100644index f882240b4cc..00000000000--- a/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.clj+++ /dev/null@@ -1,126 +0,0 @@-(ns metabase-enterprise.remote-sync.data-app-pull-test- "A remote-sync pull's outcome (the \"N changes\" summary the admin sees) must- count data-app changes. Data apps live under `data_apps/` — outside serdes — and- are materialized separately (`metabase-enterprise.data-apps.sync`), so without- `impl/fold-data-app-changes` a pull whose only changes are data apps reports- `pull-skipped` / `count 0`. These cover the three pull shapes: data-app-only,- mixed (serdes + data apps), and serdes-only."- (:require- [clojure.test :refer :all]- [metabase-enterprise.data-apps.sync :as data-apps.sync]- [metabase-enterprise.remote-sync.impl :as impl]- [metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]- [metabase-enterprise.remote-sync.source.protocol :as source.p]- [metabase-enterprise.remote-sync.test-helpers :as rs.test]- [metabase.search.test-util :as search.tu]- [metabase.test :as mt]- [metabase.test.fixtures :as fixtures]- [toucan2.core :as t2]))--(set! *warn-on-reflection* true)--(use-fixtures :once (fixtures/initialize :db))-(use-fixtures :each rs.test/clean-remote-sync-state rs.test/commit-with-temp)-(defn- import-at!- "Run `import!` against the source's snapshot at `version`, complete the task (so- `last-version` advances for the next pull), and return the result."- [src version & {:keys [force?] :or {force? false}}]- (let [task (t2/insert-returning-pk! :model/RemoteSyncTask- {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})- result (impl/import! (source.p/snapshot-at src version) task :force? force?)]- (impl/handle-task-result! result task)- result))--(defn- app-tree!- "Repo files for one data app: its `data_app.yaml` + a bundle at `dist/index.js`."- [slug bundle]- {(str "data_apps/" slug "/data_app.yaml") (str "name: " slug "\npath: dist/index.js\n")- (str "data_apps/" slug "/dist/index.js") bundle})--;; One self-contained serdes entity (a bare collection — no DB deps), used as the-;; "data" change. Reuses the path/content shape the mock harness imports cleanly.-(def ^:private coll-path- "collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/M-Q4pcV0qkiyJ0kiSWECl_some_collection.yaml")-(def ^:private coll-file- {coll-path (rs.test/generate-collection-yaml "M-Q4pcV0qkiyJ0kiSWECl" "Some Collection")})--(defn- pull-outcome!- "Establish `v0` as the baseline, then pull `v1` (a normal, non-forced pull) and- return its `:outcome`."- [v0 v1]- (let [src (rs.test/versioned-source :trees {"v0" v0 "v1" v1} :current "v0")]- (is (= :success (:status (import-at! src "v0" :force? true))) "baseline import succeeds")- (let [result (import-at! src "v1")]- (is (= :success (:status result)) "pull succeeds")- (:outcome result))))--(deftest data-app-only-pull-counts-app-changes-test- (testing "a pull whose only change is a data app is reported as a real pull, not pull-skipped / 0 changes"- (search.tu/with-index-disabled- (mt/with-premium-features #{:data-apps}- (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [outcome (pull-outcome! (app-tree! "sales" "BUNDLE-V1")- (app-tree! "sales" "BUNDLE-V2"))]- (is (= "pulled" (:kind outcome)) "not reported as skipped")- (is (= 1 (:count outcome)) "the one changed data app is counted"))))))))--(deftest data-only-pull-excludes-unchanged-apps-test- (testing "a serdes-only pull counts serdes content; unchanged data apps add nothing"- (search.tu/with-index-disabled- (mt/with-premium-features #{:data-apps}- (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- ;; v1 adds a collection; the data app is byte-for-byte the same as v0.- (let [app (app-tree! "sales" "BUNDLE")- outcome (pull-outcome! app (merge coll-file app))]- (is (= "pulled" (:kind outcome)))- (is (= 1 (:count outcome)) "the collection counts; the unchanged app adds 0"))))))))--(deftest mixed-pull-counts-serdes-and-apps-test- (testing "a mixed pull counts serdes content AND the changed data apps"- (search.tu/with-index-disabled- (mt/with-premium-features #{:data-apps}- (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- ;; v1 adds a collection AND changes the data app's bundle.- (let [outcome (pull-outcome! (app-tree! "ops" "BUNDLE")- (merge coll-file (app-tree! "ops" "BUNDLE-V2")))]- (is (= "pulled" (:kind outcome)))- (is (= 2 (:count outcome)) "the collection (1) plus the changed data app (1)"))))))))--(deftest data-apps-land-with-the-version-test- (testing "a pull records the data apps at the snapshot version it commits as the sync base"- (search.tu/with-index-disabled- (mt/with-premium-features #{:data-apps}- (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [src (rs.test/versioned-source :trees {"v0" (app-tree! "sales" "BUNDLE-V1")- "v1" (app-tree! "sales" "BUNDLE-V2")}- :current "v0")]- (is (= :success (:status (import-at! src "v0" :force? true))))- (is (= :success (:status (import-at! src "v1"))))- (is (= "v1" (remote-sync.task/last-version)))- (is (= "v1" (t2/select-one-fn :last_synced_sha :model/DataApp :name "sales"))))))))))--(deftest data-app-failure-rolls-back-the-version-test- (testing "a crash while materializing data apps rolls the version back, so the next pull redoes the import- instead of skipping a version whose data apps never landed"- (search.tu/with-index-disabled- (mt/with-premium-features #{:data-apps}- (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [src (rs.test/versioned-source :trees {"v0" (app-tree! "sales" "BUNDLE-V1")- "v1" (app-tree! "sales" "BUNDLE-V2")}- :current "v0")]- (is (= :success (:status (import-at! src "v0" :force? true))))- (let [v0-hash (t2/select-one-fn :bundle_hash :model/DataApp :name "sales")]- (mt/with-dynamic-fn-redefs [data-apps.sync/sync-from-snapshot! (fn [_] (throw (ex-info "materialization died" {})))]- (is (= :error (:status (import-at! src "v1")))))- (is (= "v0" (remote-sync.task/last-version)) "the failed pull is not the sync base")- (is (= v0-hash (t2/select-one-fn :bundle_hash :model/DataApp :name "sales"))- "the v0 app is untouched"))- (is (=? {:status :success :outcome {:kind "pulled"}} (import-at! src "v1"))- "the next pull redoes the import rather than skipping v1")- (is (= "v1" (remote-sync.task/last-version)))- (is (= "v1" (t2/select-one-fn :last_synced_sha :model/DataApp :name "sales"))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.cljindex f223099455a..42e2fd494e8 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj@@ -1,77 +1,154 @@(ns metabase-enterprise.remote-sync.data-apps-test- "Integration coverage for the data-apps hook in the remote-sync import pipeline:- importing a repo materializes data apps from its `data_apps/` directory."+ "Data apps are serdes entities: remote sync imports and exports them like any other, with each app's bundle as a+ resource file next to its `data_app.yaml`."(:require[clojure.test :refer :all]- [metabase-enterprise.data-apps.models.data-app][metabase-enterprise.remote-sync.impl :as impl]+ [metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task][metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.remote-sync.test-helpers :as test-helpers]+ [metabase.search.test-util :as search.tu][metabase.test :as mt][metabase.test.fixtures :as fixtures]+ [metabase.util.yaml :as yaml][toucan2.core :as t2]))+(set! *warn-on-reflection* true)+(use-fixtures :once (fixtures/initialize :db))-(use-fixtures :each (fn [f] (test-helpers/clean-remote-sync-state f)))-(comment metabase-enterprise.data-apps.models.data-app/keep-me)+(use-fixtures :each test-helpers/clean-remote-sync-state test-helpers/commit-with-temp)++(def ^:private sales-eid "w8YkMQBwU4kOD0-pZ1nJk")++(defn- app-tree+ "Repo files for the data app `slug` with entity id `eid`: its `data_app.yaml` and its bundle at `dist/index.js`."+ [eid slug bundle]+ {(str "data_apps/" slug "/data_app.yaml")+ (yaml/generate-string {:serdes/meta [{:model "DataApp" :id eid :label slug}]+ :entity_id eid+ :slug slug+ :name "Sales"+ :path "dist/index.js"})+ (str "data_apps/" slug "/dist/index.js") bundle})++(defn- new-task! [sync-task-type]+ (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type sync-task-type :initiated_by (mt/user->id :rasta)}))++(defn- import-at!+ "Run `import!` against the source's snapshot at `version`, complete the task (so `last-version` advances for the+ next pull), and return the result."+ [src version & {:keys [force?] :or {force? false}}]+ (let [task (new-task! "import")+ result (impl/import! (source.p/snapshot-at src version) task :force? force?)]+ (impl/handle-task-result! result task)+ result))++(defn- export!+ "Export to `mock` under a fresh task, complete it, and return the result."+ [mock]+ (t2/delete! :model/RemoteSyncTask)+ (let [task (new-task! "export")+ result (impl/export! (source.p/snapshot mock) task "export")]+ (impl/handle-task-result! result task)+ result))++(defn- bundle-text [slug]+ (when-let [^bytes bundle (t2/select-one-fn :bundle [:model/DataApp :bundle] :name slug)]+ (String. bundle "UTF-8")))++(defmacro ^:private with-data-apps-sync [& body]+ `(search.tu/with-index-disabled+ (mt/with-premium-features #{:data-apps}+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true remote-sync-transforms false]+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]+ ~@body)))))++(deftest pull-imports-and-prunes-data-apps-test+ (with-data-apps-sync+ (let [src (test-helpers/versioned-source :trees {"v0" (app-tree sales-eid "sales" "BUNDLE-V1")+ "v1" {"README.md" "x"}}+ :current "v0")]+ (testing "a pull imports the app and its bundle"+ (is (=? {:status :success :outcome {:kind "pulled" :count 1}} (import-at! src "v0" :force? true)))+ (is (=? {:entity_id sales-eid :display_name "Sales" :draft false :resource_collection_id pos-int?}+ (t2/select-one :model/DataApp :name "sales")))+ (is (= "BUNDLE-V1" (bundle-text "sales"))))+ (testing "a pull whose repo no longer has the app removes it"+ (is (= :success (:status (import-at! src "v1"))))+ (is (not (t2/exists? :model/DataApp :name "sales")))))))-(defn- import! [files]- ;; the import's data-app materialization is gated on :data-apps; enable it so apps sync.- (mt/with-premium-features #{:data-apps}- (let [source (test-helpers/create-mock-source :initial-files files)- task-id (t2/insert-returning-pk! :model/RemoteSyncTask- {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})- ;; force: the mock source reports a constant version, so without this the- ;; second import in a test would hit the "version unchanged" skip branch- result (impl/import! (source.p/snapshot source) task-id :force? true)]- ;; calling import! directly leaves the task "running"; mark it ended so a- ;; subsequent import in the same test isn't blocked by the running-task guard- (t2/update! :model/RemoteSyncTask task-id {:ended_at :%now})- result)))+(deftest bundle-only-pull-updates-the-bundle-test+ (testing "a pull that changes only an app's bundle file falls back to a full import, so the new bundle lands"+ (with-data-apps-sync+ (let [src (test-helpers/versioned-source :trees {"v0" (app-tree sales-eid "sales" "BUNDLE-V1")+ "v1" (app-tree sales-eid "sales" "BUNDLE-V2")}+ :current "v0")]+ (is (= :success (:status (import-at! src "v0" :force? true))))+ (is (=? {:status :success :outcome {:kind "pulled"}} (import-at! src "v1")))+ (is (= "v1" (remote-sync.task/last-version)))+ (is (= "BUNDLE-V2" (bundle-text "sales")))))))-(deftest import-materializes-data-apps-test- (testing "a remote-sync import materializes data apps from data_apps/ alongside serdes content"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (let [files {"main" {"collections/c/c.yaml"- (test-helpers/generate-collection-yaml "data-apps-test-collx" "DA Coll")- "data_apps/sales/data_app.yaml" "name: Sales\npath: ./dist/index.js\n"- "data_apps/sales/dist/index.js" "SALESBUNDLE"}}- result (import! files)]- (is (= :success (:status result)))- (let [app (t2/select-one :model/DataApp :name "sales")]- (is (some? app) "the data app from the repo was materialized by the import")- (is (= "Sales" (:display_name app)))- (is (= "data_apps/sales/dist/index.js" (:bundle_path app)))- (is (true? (:enabled app)))- (is (= "SALESBUNDLE" (String. ^bytes (:bundle app) "UTF-8"))))))))+(deftest export-writes-and-removes-data-app-files-test+ (with-data-apps-sync+ (let [mock (test-helpers/create-mock-source :initial-files {"main" {}})+ repo #(get @(:files-atom mock) "main")]+ (mt/user-http-request :crowberto :post 200 "apps" {:name "sales"+ :display_name "Sales"+ :bundle_path "dist/index.js"+ :bundle "BUNDLE"})+ (testing "a created app is exported as its manifest and its bundle file"+ (is (= :success (:status (export! mock))))+ (is (=? {:slug "sales" :name "Sales" :path "dist/index.js"}+ (yaml/parse-string (get (repo) "data_apps/sales/data_app.yaml"))))+ (is (= "BUNDLE" (get (repo) "data_apps/sales/dist/index.js"))))+ (testing "an updated bundle is exported"+ (mt/user-http-request :crowberto :put 200 "apps/sales" {:bundle "BUNDLE-V2"})+ (is (= :success (:status (export! mock))))+ (is (= "BUNDLE-V2" (get (repo) "data_apps/sales/dist/index.js"))))+ (testing "a deleted app's files are removed from the repo, and the other apps' files stay"+ (mt/user-http-request :crowberto :post 200 "apps" {:name "ops"+ :display_name "Ops"+ :bundle_path "app.js"+ :bundle "OPS"})+ (mt/user-http-request :crowberto :delete 204 "apps/sales")+ (is (= :success (:status (export! mock))))+ (is (= #{"data_apps/ops/data_app.yaml" "data_apps/ops/app.js"}+ (into #{} (filter #(re-find #"^data_apps/" %)) (keys (repo)))))))))-(deftest import-prunes-data-apps-absent-from-repo-test- (testing "an import whose repo no longer has an app dir prunes that app (the repo is the source of truth)"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- (import! {"main" {"data_apps/gone/data_app.yaml" "name: Gone\npath: ./i.js\n"- "data_apps/gone/i.js" "X"- "data_apps/kept/data_app.yaml" "name: Kept\npath: ./i.js\n"- "data_apps/kept/i.js" "K"}})- (is (= #{"gone" "kept"} (t2/select-fn-set :name :model/DataApp)))- (import! {"main" {"data_apps/kept/data_app.yaml" "name: Kept\npath: ./i.js\n"- "data_apps/kept/i.js" "K"}})- (is (nil? (t2/select-one :model/DataApp :name "gone"))- "the app absent from the later import is pruned")- (is (some? (t2/select-one :model/DataApp :name "kept"))- "the still-present app is kept"))))+(deftest export-keeps-files-serialization-does-not-own-test+ (testing "an export rewrites an app's manifest and bundle but leaves the app's source next to them alone"+ (with-data-apps-sync+ (let [source {"data_apps/sales/src/App.tsx" "export const App = () => null;"+ "data_apps/sales/package.json" "{}"+ "data_apps/sales/pnpm-lock.yaml" "lockfileVersion: '9.0'\n"+ "data_apps/sales/deploy/k8s.yaml" "a: 1\n---\nb: [unterminated\n"}+ mock (test-helpers/create-mock-source+ :initial-files {"main" (merge (app-tree sales-eid "sales" "BUNDLE-V1") source)})+ repo #(get @(:files-atom mock) "main")]+ (is (= :success (:status (import-at! mock "main" :force? true))))+ (mt/user-http-request :crowberto :put 200 "apps/sales" {:bundle "BUNDLE-V2"})+ (mt/user-http-request :crowberto :post 200 "apps" {:name "ops"+ :display_name "Ops"+ :bundle_path "app.js"+ :bundle "OPS"})+ (is (= :success (:status (export! mock))))+ (is (= "BUNDLE-V2" (get (repo) "data_apps/sales/dist/index.js")))+ (is (= source (select-keys (repo) (keys source))))+ (testing "deleting the app removes only the files serialization owns"+ (mt/user-http-request :crowberto :delete 204 "apps/sales")+ (is (= :success (:status (export! mock))))+ (is (= (into #{"data_apps/ops/data_app.yaml" "data_apps/ops/app.js"} (keys source))+ (into #{} (filter #(re-find #"^data_apps/" %)) (keys (repo))))))))))-(deftest deletion-only-pull-counts-the-removal-test- (testing "a pull whose only change is removing an app still reports as a pull — the removal is counted, not a no-op"- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]- ;; No serdes content, so the outcome's count comes purely from data apps.- (import! {"main" {"README.md" "x"- "data_apps/gone/data_app.yaml" "name: Gone\npath: ./i.js\n"- "data_apps/gone/i.js" "X"}})- (is (= #{"gone"} (t2/select-fn-set :name :model/DataApp)))- ;; Same repo minus the app dir: nothing is upserted (`:changed` 0), one app removed.- (let [result (import! {"main" {"README.md" "x"}})]- (is (= :success (:status result)))- (is (empty? (t2/select-fn-set :name :model/DataApp))- "the app is pruned")- (is (pos? (:count (:outcome result)))- "the removal is folded into the pull outcome, so it isn't reported as pull-skipped/0")))))+(deftest pull-refuses-to-delete-an-unpushed-app-test+ (testing "a pull whose repo lacks an app created here but not pushed yet is a conflict, not a silent delete"+ (with-data-apps-sync+ (let [app (app-tree sales-eid "sales" "BUNDLE")+ src (test-helpers/versioned-source :trees {"v0" app "v1" (assoc app "README.md" "x")} :current "v0")]+ (is (= :success (:status (import-at! src "v0" :force? true))))+ (mt/user-http-request :crowberto :post 200 "apps" {:name "ops"+ :display_name "Ops"+ :bundle_path "app.js"+ :bundle "OPS"})+ (is (=? {:status :conflict} (import-at! src "v1")))+ (is (t2/exists? :model/DataApp :name "ops"))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.cljindex 249dc172836..5afdf0d3589 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj@@ -44,7 +44,7 @@(deftest all-specs-have-valid-eligibility-test(testing "Every spec has a valid eligibility type"- (let [valid-eligibility-types #{:collection :published-table :parent-table :parent :setting :library-synced}]+ (let [valid-eligibility-types #{:collection :published-table :parent-table :parent :setting :library-synced :always}](doseq [[model-key spec] spec/remote-sync-specs](testing (str "Spec for " model-key)(is (contains? valid-eligibility-types (get-in spec [:eligibility :type]))@@ -137,7 +137,8 @@(is (contains? types "TransformTest"))(is (contains? types "Glossary"))(is (contains? types "Action"))- (is (= 16 (count types))))))+ (is (contains? types "DataApp"))+ (is (= 17 (count types))))))(deftest specs-by-identity-type-test(testing "specs-by-identity-type filters correctly"diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex 2b49d5e3fc4..bf9f55be513 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -267,7 +267,7 @@ width: fixed:or {branch "main"fail-mode nilinitial-files nil- managed-dirs ingest/legal-top-level-paths}}]+ managed-dirs ingest/replaced-top-level-paths}}](let [default-files {"main" {"collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/M-Q4pcV0qkiyJ0kiSWECl_some_collection.yaml"(generate-collection-yaml "M-Q4pcV0qkiyJ0kiSWECl" "Some Collection")@@ -297,7 +297,7 @@ width: fixedmodel a base orphaned by a force-push/rebase). Committing (via `open-commit`) records the written setunder a fresh version, advances `:current`, and returns the new version so an export can fast-forward onto it."[& {:keys [current trees branch managed-dirs]- :or {current "v-remote" branch "main" managed-dirs ingest/legal-top-level-paths}}]+ :or {current "v-remote" branch "main" managed-dirs ingest/replaced-top-level-paths}}](let [managed (set managed-dirs)state (atom {:current current :trees (or trees {}) :counter 0})diff-trees (fn [old-tree new-tree]diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.cljindex 76f1959b6d3..5131b320585 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj@@ -54,7 +54,6 @@:model/CollectionBookmark:model/ContentTranslation:model/DashboardBookmark- :model/DataApp:model/DataComplexityScore:model/DataPermissions:model/DatabaseRouterdiff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.cljindex 74a11b1e898..695147eecd6 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj@@ -148,10 +148,11 @@fixture in this shared baseline. OsiAiContext is covered (in-memory + on-disk) bymetabase-enterprise.serialization.v2.osi-ai-context-test. TransformTest is covered bytransform-test-round-trip-test and transform-test-expectations-round-trip-test in- metabase-enterprise.serialization.v2.e2e-test, and stays out of the baseline until- @metabase/representations publishes a schema for it — the baseline is validated against that+ metabase-enterprise.serialization.v2.e2e-test, and DataApp by+ metabase-enterprise.data-apps.serialization-test; both stay out of the baseline until+ @metabase/representations publishes a schema for them — the baseline is validated against thatpackage, which refuses a model it does not know."- #{"OsiAiContext" "TransformTest"})+ #{"DataApp" "OsiAiContext" "TransformTest"})(defn add-to-baseline!"Use this within v2.extract-test where relevant to add their fixtures to the baseline."diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.tsindex f2348484c48..03c6d6c0fc5 100644--- a/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.ts+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.ts@@ -4,6 +4,7 @@ import path from "node:path";import { load as parseYaml } from "js-yaml";export type DataAppManifest = {+ slug?: string;allowed_hosts?: string[];};@@ -34,6 +35,21 @@ const parseAllowedHosts = (return value;};+const parseSlug = (+ value: unknown,+ manifestPath: string,+): string | undefined => {+ if (value == null) {+ return undefined;+ }++ if (!isString(value)) {+ throw new Error(`${manifestPath}: "slug" must be a string.`);+ }++ return value;+};+export const readManifest = (appRoot: string,): { manifestPath: string; manifest: DataAppManifest } | null => {@@ -57,12 +73,13 @@ export const readManifest = ();}- const raw: { allowed_hosts?: unknown } =+ const raw: { slug?: unknown; allowed_hosts?: unknown } =typeof parsed === "object" && parsed !== null ? parsed : {};return {manifestPath,manifest: {+ slug: parseSlug(raw.slug, manifestPath),allowed_hosts: parseAllowedHosts(raw.allowed_hosts, manifestPath),},};diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.unit.spec.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.unit.spec.tsindex a71b090e99d..beed6289a4c 100644--- a/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.unit.spec.ts+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.unit.spec.ts@@ -44,6 +44,20 @@ describe("readManifest", () => {expect(() => readManifest(APP_ROOT)).toThrow(/Could not parse/);});+ describe("slug", () => {+ it("reads the slug", () => {+ setup({ [YAML_PATH]: "slug: sales-ops\n" });++ expect(readManifest(APP_ROOT)?.manifest.slug).toBe("sales-ops");+ });++ it("throws on a non-string slug", () => {+ setup({ [YAML_PATH]: "slug: [a]\n" });++ expect(() => readManifest(APP_ROOT)).toThrow(/"slug" must be a string/);+ });+ });+describe("allowed_hosts", () => {it("reads allowed_hosts as a list of strings", () => {setup({diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/validate-manifest.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/validate-manifest.tsindex 0fc57d76cf2..9c4919d4091 100644--- a/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/validate-manifest.ts+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/validate-manifest.ts@@ -36,7 +36,7 @@ const compareAllowedHosts = (};/**- * Not a full validator: the manifest rules live in the backend's `parse-app-config`+ * Not a full validator: the manifest rules live in the backend's data-apps schemas* and a copy here would drift. This only adds what the backend cannot see —* this machine's disk, and what the dev server booted with.*/diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/sync.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/sync.tsindex 0f26582cfe3..fc65ac1c0e9 100644--- a/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/sync.ts+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/sync.ts@@ -1,5 +1,7 @@import path from "node:path";+import { readManifest } from "../data-app-dev/config/read-manifest";+import { discoverActions, discoverQueries } from "./discover";import { isPositiveInteger } from "./guards";import {@@ -161,7 +163,12 @@ export async function syncResources({]);const lockfile = readResourceLockfile(appRoot);const client = new MetabaseClient(metabaseUrl, apiKey);- const slug = path.basename(appRoot);+ const slug = readManifest(appRoot)?.manifest.slug;+ if (!slug) {+ throw new Error(+ `${path.join(appRoot, "data_app.yaml")} must declare the app's "slug".`,+ );+ }const app = await client.ensureDraft(slug);if (!isPositiveInteger(app.resource_collection_id)) {throw new Error(`Data app ${slug} does not have a resource collection.`);diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/setup.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/setup.tsindex 58fd7e8868a..a8919b6cee7 100644--- a/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/setup.ts+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/setup.ts@@ -27,6 +27,10 @@ export function makeApp() {fs.mkdirSync(path.join(appRoot, "queries"));fs.mkdirSync(path.join(appRoot, "actions"));+ fs.writeFileSync(+ path.join(appRoot, "data_app.yaml"),+ `slug: ${path.basename(appRoot)}\n`,+ );const packageRoot = path.join(appRoot,diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/sync.unit.spec.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/sync.unit.spec.tsindex 3cf88b7187a..f81667f096f 100644--- a/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/sync.unit.spec.ts+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/sync.unit.spec.ts@@ -1,3 +1,4 @@+import fs from "node:fs";import path from "node:path";import { discoverQueries } from "../discover";@@ -14,6 +15,45 @@ import {describe("query synchronization", () => {setupResourceSyncTests();+ it("synchronizes the app named by the manifest's slug, not its directory", async () => {+ const appRoot = makeApp();+ fs.writeFileSync(path.join(appRoot, "data_app.yaml"), "slug: other-app\n");+ const fetchSpy = jest.spyOn(global, "fetch").mockResolvedValue(+ new Response("Not found.", {+ status: 404,+ headers: { "content-type": "text/plain" },+ }),+ );++ await expect(+ syncResources({+ appRoot,+ metabaseUrl: "http://metabase.test",+ apiKey: "secret",+ log: jest.fn(),+ }),+ ).rejects.toThrow(/apps\/other-app\/draft/);+ expect(String(fetchSpy.mock.calls[0][0])).toBe(+ "http://metabase.test/api/apps/other-app/draft",+ );+ });++ it("refuses an app whose manifest declares no slug", async () => {+ const appRoot = makeApp();+ fs.writeFileSync(path.join(appRoot, "data_app.yaml"), "name: App\n");+ const fetchSpy = jest.spyOn(global, "fetch");++ await expect(+ syncResources({+ appRoot,+ metabaseUrl: "http://metabase.test",+ apiKey: "secret",+ log: jest.fn(),+ }),+ ).rejects.toThrow(/must declare the app's "slug"/);+ expect(fetchSpy).not.toHaveBeenCalled();+ });+it("identifies the request that failed", async () => {const appRoot = makeApp();const slug = path.basename(appRoot);diff --git a/enterprise/frontend/src/metabase-enterprise/data_apps/admin/ManageDataAppsPage.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/data_apps/admin/ManageDataAppsPage.unit.spec.tsxindex 1d5b4cf9411..5b499d3bb34 100644--- a/enterprise/frontend/src/metabase-enterprise/data_apps/admin/ManageDataAppsPage.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/data_apps/admin/ManageDataAppsPage.unit.spec.tsx@@ -149,34 +149,16 @@ describe("ManageDataAppsPage", () => {).toBeInTheDocument();});- it("renders each sync status: a synced sha, a failure with its reason, and never-synced", async () => {+ it("labels drafts", async () => {setup({apps: [- createMockDataApp({- id: 1,- display_name: "Synced App",- last_synced_sha: "abcdef0",- }),- createMockDataApp({- id: 2,- display_name: "Failed App",- sync_error: "boom: bad manifest",- }),- createMockDataApp({- id: 3,- display_name: "New App",- last_synced_sha: null,- sync_error: null,- }),+ createMockDataApp({ id: 1, display_name: "Live App" }),+ createMockDataApp({ id: 2, display_name: "Draft App", draft: true }),],});- expect(await screen.findByText("Synced abcdef0")).toBeInTheDocument();- expect(screen.getByText("Sync failed")).toHaveAttribute(- "title",- "boom: bad manifest",- );- expect(screen.getByText("Not synced yet")).toBeInTheDocument();+ expect(await screen.findByText("Draft App")).toBeInTheDocument();+ expect(screen.getAllByText("Draft")).toHaveLength(1);});it("shows how many hosts an app is allowed to reach", async () => {diff --git a/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.tsx b/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.tsxindex ce5ed024514..fd1c2a0bf26 100644--- a/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.tsx+++ b/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.tsx@@ -1,7 +1,6 @@import { t } from "ttag";import { Group, Stack, Text } from "metabase/ui";-import type { MetabaseColorKey } from "metabase/ui/colors/types";import * as Urls from "metabase/urls";import type { DataApp } from "metabase-types/api";@@ -18,37 +17,8 @@ const Bullet = () => (</Text>);-function getSyncStatus({ sync_error, last_synced_sha }: DataApp): {- label: string;- color: MetabaseColorKey;- title?: string;-} {- if (sync_error) {- return { label: t`Sync failed`, color: "error", title: sync_error };- }-- if (last_synced_sha) {- return {- label: t`Synced ${last_synced_sha.slice(0, 7)}`,- color: "text-secondary",- };- }-- return { label: t`Not synced yet`, color: "text-tertiary" };-}--const SyncStatus = ({ app }: Props) => {- const { label, color, title } = getSyncStatus(app);-- return (- <Text size="sm" c={color} title={title} lh="1.4">- {label}- </Text>- );-};-export const DataAppSummary = ({ app }: Props) => {- const isOpenable = app.enabled && !app.sync_error && !app.outdated;+ const isOpenable = app.enabled && !app.draft && !app.outdated;return (<Group align="center" flex="1" wrap="nowrap" miw={0}>@@ -92,9 +62,15 @@ export const DataAppSummary = ({ app }: Props) => {{Urls.dataApp(app.name)}</Text>- <Bullet />+ {app.draft && (+ <>+ <Bullet />- <SyncStatus app={app} />+ <Text size="sm" c="text-tertiary" lh="1.4">+ {t`Draft`}+ </Text>+ </>+ )}{app.allowed_hosts.length > 0 && (<>diff --git a/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.unit.spec.tsxindex 3b502fd17da..8684bb145a4 100644--- a/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.unit.spec.tsx@@ -31,13 +31,13 @@ describe("DataAppSummary", () => {).not.toBeInTheDocument();});- it("renders a sync-failed app's name as plain text (the link would lead to a dead end)", () => {+ it("renders a draft's name as plain text (it has no bundle to open)", () => {renderWithProviders(<DataAppSummaryapp={createMockDataApp({display_name: "Sales",enabled: true,- sync_error: "Bundle file not found",+ draft: true,})}/>,);@@ -93,54 +93,19 @@ describe("DataAppSummary", () => {});});- describe("sync status", () => {- it("shows the short synced SHA when the app has synced", () => {+ describe("draft badge", () => {+ it("labels a draft", () => {renderWithProviders(- <DataAppSummary- app={createMockDataApp({- last_synced_sha: "0123456789abcdef",- sync_error: null,- })}- />,- );-- expect(screen.getByText("Synced 0123456")).toBeInTheDocument();- });-- it("shows a failure label with the error as a tooltip when the sync failed", () => {- renderWithProviders(- <DataAppSummary- app={createMockDataApp({ sync_error: "boom: bad manifest" })}- />,- );-- const status = screen.getByText("Sync failed");- expect(status).toBeInTheDocument();- expect(status).toHaveAttribute("title", "boom: bad manifest");- });-- it("prefers the failure label over the SHA when both are present", () => {- renderWithProviders(- <DataAppSummary- app={createMockDataApp({- last_synced_sha: "0123456789abcdef",- sync_error: "boom",- })}- />,+ <DataAppSummary app={createMockDataApp({ draft: true })} />,);- expect(screen.getByText("Sync failed")).toBeInTheDocument();- expect(screen.queryByText(/^Synced/)).not.toBeInTheDocument();+ expect(screen.getByText("Draft")).toBeInTheDocument();});- it("shows 'Not synced yet' when the app has never synced", () => {- renderWithProviders(- <DataAppSummary- app={createMockDataApp({ last_synced_sha: null, sync_error: null })}- />,- );+ it("doesn't label an app that isn't a draft", () => {+ renderWithProviders(<DataAppSummary app={createMockDataApp()} />);- expect(screen.getByText("Not synced yet")).toBeInTheDocument();+ expect(screen.queryByText("Draft")).not.toBeInTheDocument();});});});diff --git a/frontend/src/metabase-types/api/data-app.ts b/frontend/src/metabase-types/api/data-app.tsindex 899524da0ef..b3337375b8a 100644--- a/frontend/src/metabase-types/api/data-app.ts+++ b/frontend/src/metabase-types/api/data-app.ts@@ -1,13 +1,13 @@export type DataAppId = number;/**- * A data app materialized from the connected repository. The repo's- * `data_apps/<dir>/data_app.yaml` files are discovered on sync; one row exists- * per app and the bundle is cached in the app DB and served at `/apps/:name`.- * The repository itself is configured via the remote-sync settings.+ * A data app: its manifest fields, plus a bundle cached in the app DB and+ * served at `/apps/:name`.*/export interface DataApp {id: DataAppId;+ /** Stable identifier shared by the app's copies across instances. */+ entity_id: string;/** The app's slug. */name: string;display_name: string;@@ -20,10 +20,12 @@ export interface DataApp {* regular users never receive an outdated app.*/outdated: boolean;- /** Path within the repo to the built bundle. */+ /** Path to the built bundle, relative to the app's directory. */bundle_path: string;/** Admin toggle. When false the app is not served. */enabled: boolean;+ /** Whether the app is a draft that reserves its slug before the app is created. */+ draft: boolean;/** The collection that contains this app's saved questions and models. */resource_collection_id: number | null;/** The group that grants users access to this data app. */@@ -33,19 +35,13 @@ export interface DataApp {/** Whether any app member lacks access to a table used by this app. */has_user_permission_warnings?: boolean;/**- * External origins the app's sandboxed bundle may `fetch`/XHR, from its- * `data_app.yaml`. Empty means none (Metabase data still flows through the- * SDK). Each entry is an origin, optionally with a `*.` wildcard.+ * External origins the app's sandboxed bundle may `fetch`/XHR. Empty means+ * none (Metabase data still flows through the SDK). Each entry is an origin,+ * optionally with a `*.` wildcard.*/allowed_hosts: string[];- /** SHA-256 of the cached bundle; `null` until the first successful sync. */+ /** SHA-256 of the cached bundle; `null` while the app has none. */bundle_hash: string | null;- /** Git commit the cached bundle was synced from; `null` until first sync. */- last_synced_sha: string | null;- /** ISO timestamp of the last successful sync; `null` if never synced. */- last_synced_at: string | null;- /** Message from this app's most recent failed sync; `null` when it succeeded. */- sync_error: string | null;created_at: string;updated_at: string;}diff --git a/frontend/src/metabase-types/api/mocks/data-app.ts b/frontend/src/metabase-types/api/mocks/data-app.tsindex e19d7a03e20..12d9c48033f 100644--- a/frontend/src/metabase-types/api/mocks/data-app.ts+++ b/frontend/src/metabase-types/api/mocks/data-app.ts@@ -2,22 +2,21 @@ import type { DataApp } from "metabase-types/api";export const createMockDataApp = (opts?: Partial<DataApp>): DataApp => ({id: 1,+ entity_id: "mOcKdAtAaPpEnTiTyIdXx",name: "sales",display_name: "Sales",description: null,version: 1,outdated: false,- bundle_path: "data_apps/sales/dist/index.js",+ bundle_path: "dist/index.js",enabled: true,+ draft: false,resource_collection_id: 1,permission_group_id: 1,table_ids: [],has_user_permission_warnings: false,allowed_hosts: [],bundle_hash: "abc123",- last_synced_sha: "0123456789abcdef",- last_synced_at: "2024-01-01T00:00:00Z",- sync_error: null,created_at: "2024-01-01T00:00:00Z",updated_at: "2024-01-01T00:00:00Z",...opts,diff --git a/resources/migrations/065/20261001_data_app_serialization.yaml b/resources/migrations/065/20261001_data_app_serialization.yamlnew file mode 100644index 00000000000..9b9a43a459b--- /dev/null+++ b/resources/migrations/065/20261001_data_app_serialization.yaml@@ -0,0 +1,121 @@+## Data apps become regular serdes entities. They get an entity_id so serialization and remote sync can key them on a+## stable, portable identifier; the backfill mirrors 20260911_glossary_entity_id: all three dialect variants in one+## changeset, then NOT NULL and UNIQUE. Imports now fail as a whole rather than per app, so the per-app sync+## columns go away.++databaseChangeLog:+ - changeSet:+ id: v65.2026-10-01T00:00:00+ author: ranquild+ comment: Add entity_id column to data_app table+ changes:+ - addColumn:+ tableName: data_app+ columns:+ - column:+ name: entity_id+ type: char(21)+ remarks: Random NanoID tag for unique identity.+ constraints:+ nullable: true+ rollback:+ - dropColumn:+ tableName: data_app+ columnName: entity_id++ - changeSet:+ id: v65.2026-10-01T00:00:01+ author: ranquild+ comment: Backfill NULL data_app.entity_id values+ changes:+ - sql:+ dbms: postgresql+ sql: UPDATE data_app SET entity_id = substr(replace(gen_random_uuid()::text, '-', ''), 1, 21) WHERE entity_id IS NULL;+ - sql:+ dbms: mysql,mariadb+ sql: UPDATE data_app SET entity_id = substr(replace(uuid(), '-', ''), 1, 21) WHERE entity_id IS NULL;+ - sql:+ dbms: h2+ sql: UPDATE data_app SET entity_id = substr(replace(cast(random_uuid() as varchar), '-', ''), 1, 21) WHERE entity_id IS NULL;+ rollback: []++ - changeSet:+ id: v65.2026-10-01T00:00:02+ author: ranquild+ comment: entity_id is NOT NULL on data_app+ changes:+ - addNotNullConstraint:+ tableName: data_app+ columnName: entity_id+ columnDataType: char(21)++ - changeSet:+ id: v65.2026-10-01T00:00:03+ author: ranquild+ comment: entity_id is unique on data_app+ changes:+ - addUniqueConstraint:+ tableName: data_app+ columnNames: entity_id+ constraintName: uq_data_app_entity_id+ rollback:+ - dropUniqueConstraint:+ tableName: data_app+ constraintName: uq_data_app_entity_id++ - changeSet:+ id: v65.2026-10-01T00:00:04+ author: ranquild+ comment: Drop data_app.sync_error, which only the removed per-app sync wrote+ changes:+ - dropColumn:+ tableName: data_app+ columnName: sync_error+ rollback:+ - addColumn:+ tableName: data_app+ columns:+ - column:+ name: sync_error+ type: ${text.type}+ remarks: Error from the most recent failed sync of this app, NULL when the last sync succeeded+ constraints:+ nullable: true++ - changeSet:+ id: v65.2026-10-01T00:00:05+ author: ranquild+ comment: Drop data_app.last_synced_sha, which only the removed per-app sync wrote+ changes:+ - dropColumn:+ tableName: data_app+ columnName: last_synced_sha+ rollback:+ - addColumn:+ tableName: data_app+ columns:+ - column:+ name: last_synced_sha+ type: varchar(40)+ remarks: Git commit SHA the app was last synced from+ constraints:+ nullable: true++ - changeSet:+ id: v65.2026-10-01T00:00:06+ author: ranquild+ comment: Drop data_app.last_synced_at, which only the removed per-app sync wrote+ changes:+ - dropColumn:+ tableName: data_app+ columnName: last_synced_at+ rollback:+ - addColumn:+ tableName: data_app+ columns:+ - column:+ name: last_synced_at+ type: ${timestamp_type}+ remarks: Timestamp of the last successful sync of this app+ constraints:+ nullable: truediff --git a/skills/metabase-data-app-migrate/SKILL.md b/skills/metabase-data-app-migrate/SKILL.mdindex f368170018c..f8f1eee6e40 100644--- a/skills/metabase-data-app-migrate/SKILL.md+++ b/skills/metabase-data-app-migrate/SKILL.md@@ -143,7 +143,7 @@ Do not batch steps across upgrades. Do not touch `version` before item 5.```bash( source "$ROOT/.env.local"; curl -s -H "x-api-key: $DATA_APP_MB_API_KEY" "$DATA_APP_MB_URL/api/apps/<slug>" )```- must show `"version": M`, `"outdated": false`, and `"sync_error": null`.+ must show `"version": M` and `"outdated": false`.A response holding only `name` and `display_name` means the key is not anadmin's: ask for one, or ask the user to read the admin list, and say theproof is still pending. Report the command and the result. Do not claim thediff --git a/skills/metabase-data-app-migrate/references/upgrade-guide-template.md b/skills/metabase-data-app-migrate/references/upgrade-guide-template.mdindex f0735c940c1..228ac28ec35 100644--- a/skills/metabase-data-app-migrate/references/upgrade-guide-template.md+++ b/skills/metabase-data-app-migrate/references/upgrade-guide-template.md@@ -91,7 +91,7 @@ ends with a **Done when** that reads the API:>Done when:```bash-( source "$ROOT/.env.local"; curl -s -H "x-api-key: $DATA_APP_MB_API_KEY" "$DATA_APP_MB_URL/api/apps/<slug>" ) | grep -q '"sync_error":null' && echo ok+( source "$ROOT/.env.local"; curl -s -H "x-api-key: $DATA_APP_MB_API_KEY" "$DATA_APP_MB_URL/api/apps/<slug>" ) | grep -q '"outdated":false' && echo ok```## Done-check summarydiff --git a/skills/metabase-data-app-setup/SKILL.md b/skills/metabase-data-app-setup/SKILL.mdindex 9d8900b439d..84674b6d6af 100644--- a/skills/metabase-data-app-setup/SKILL.md+++ b/skills/metabase-data-app-setup/SKILL.md@@ -7,7 +7,7 @@ description: Scaffold a new Metabase data-app into the connected remote-sync repA Metabase **data-app** is a single JS bundle that the host loads inside a Near Membrane sandbox and renders inside its own React tree. The scaffold is a Vite + React + TypeScript project: source under `src/`, a dev server that previews the app against a real Metabase **through the same Near Membrane sandbox + distortion rules Metabase uses in production** — so `npm run dev` behaves like production, including for third-party libraries the app bundles — and `npm run build` producing a single `dist/index.js`. (Because the sandbox runs a built bundle, a change rebuilds it and does a *soft reload* — re-evaluates the bundle in the sandbox and remounts the app, keeping auth/SDK loaded — rather than hot-swapping modules; component state resets, but there's no full browser refresh.) The dev preview also shows a corner **⚠ Diagnostics** toolbar that captures runtime errors — including the sandbox's otherwise-opaque blocked-API messages — so failures surface instead of being swallowed. The same data is served as JSON at `http://localhost:5174/__data-app/diagnostics`, which is how *you* read it (see "Reading the diagnostics feed" below) — you have a shell, not a browser, and these failures are invisible from the terminal otherwise.-**Data apps are served from Git, not uploaded.** A single repository is connected to Metabase via remote-sync (Admin → Settings → Remote sync). Each app lives in its own directory `data_apps/<app>/` inside that repo — its source, a `data_app.yaml` (name/path), and the committed built bundle at the `path` its `data_app.yaml` declares (`dist/index.js` by default). On each remote-sync import Metabase materializes one app per directory and serves it at `/apps/<slug>` url, where the slug **is** the directory's name. So this skill always scaffolds **into the connected repo's `data_apps/<app>/` directory**, never as a standalone project.+**Data apps are served from Git** (they can also be created and updated through `/api/apps`). A single repository is connected to Metabase via remote-sync (Admin → Settings → Remote sync). Each app lives in its own directory `data_apps/<slug>/` inside that repo — its source, a `data_app.yaml` (slug/name/path), and the committed built bundle at the `path` its `data_app.yaml` declares (`dist/index.js` by default). On each remote-sync import Metabase materializes each app and serves it at `/apps/<slug>`, where the slug is the `slug` its `data_app.yaml` declares. So this skill always scaffolds **into the connected repo's `data_apps/<slug>/` directory**, never as a standalone project.**The scaffold ships inside this skill at `./template/`** — a Vite + React + TypeScript project that was installed alongside the skill. Step 3 just copies it into the app directory; the skill then guides you through the customization + first-app-content steps — it never generates project files from scratch. If you find yourself writing `package.json`, `vite.config.ts`, `tsconfig.json`, or `src/index.tsx` by hand, stop — copy the template instead.@@ -33,7 +33,7 @@ Data apps live inside the Git repository connected to Metabase via remote-sync.## Step 2 — Name the app and create its directory-1. Settle on the app's **directory name** before scaffolding — it is used verbatim as the slug (the `/apps/<slug>` URL), so it **must be dash-cased**: lowercase letters, numbers, and single dashes (`[a-z0-9]+(?:-[a-z0-9]+)*`), e.g. `sales-overview`. Anything else (uppercase, spaces, underscores) is rejected on sync. If the purpose isn't clear yet, ask a one-line "what's this app for?" and propose a name; confirm it.+1. Settle on the app's **slug** before scaffolding — the `/apps/<slug>` URL — so it **must be dash-cased**: lowercase letters, numbers, and single dashes (`[a-z0-9]+(?:-[a-z0-9]+)*`), e.g. `sales-overview`. Anything else (uppercase, spaces, underscores) is rejected on sync. If the purpose isn't clear yet, ask a one-line "what's this app for?" and propose a slug; confirm it.2. Ensure `<repo>/data_apps/` exists; create it if missing.3. Create `<repo>/data_apps/<slug>/`. **If it already exists**, treat it as an existing project (see below) — never overwrite without confirmation.@@ -131,16 +131,22 @@ Once the template is in `<repo>/data_apps/<slug>/` (run everything below from th6. **The lockfile and the built bundle must both be committed.** Metabase serves the file at the `path` declared in `data_app.yaml` (the template builds to `dist/index.js`) straight from the committed Git tree, and the lockfile keeps installs reproducible. **Verify with `git status`** after `npm install` and a build: both must appear as committable files.7. `npm run dev` and confirm the preview at http://localhost:5174 renders the starter "Hello, data app" message.8. If the preview hits CORS, add `http://localhost:5174` under Admin → Embedding → Embedded analytics SDK → CORS.-9. **Edit `data_app.yaml`** (it ships with the template, in the app directory). This is the per-app config Metabase reads on sync — one file per app. Fill in its fields for this app:+9. **Edit `data_app.yaml`** (it ships with the template, in the app directory). This is the per-app config Metabase reads on sync — one file per app. Generate a new entity id with `node -e "console.log(require('crypto').randomBytes(16).toString('base64url').slice(0, 21))"` and fill in its fields for this app:```yaml+ version: 1 # data app contract version — leave as-is (see below)name: Sales App # display name shown in the admin UI+ slug: sales-app # the /apps/<slug> URLdescription: Pipeline health and quota attainment by region # optional — see below- version: 1 # data app contract version — leave as-is (see below)path: ./dist/index.js # bundle path, relative to this app's directory — leave as-is unless you change the build output# allowed_hosts: # optional — external origins the app may fetch/XHR (see below)# - https://api.example.com# - https://*.internal.acme.com+ entity_id: Xq2v9LbN0mTz4wRk7YsJd # the generated entity id+ serdes/meta:+ - model: DataApp+ id: Xq2v9LbN0mTz4wRk7YsJd+ label: sales-app```Commit it alongside the built bundle (the file `path` points at).@@ -524,7 +530,7 @@ Do not wrap `InteractiveQuestion` or `StaticQuestion` in containers that clip or## Sync to Metabase-Data apps are delivered by Git, not uploaded — you commit the app directory and Metabase pulls it on its next remote-sync import.+Data apps are delivered by Git — you commit the app directory and Metabase pulls it on its next remote-sync import.1. `npm run build` → produces the bundle at your `data_app.yaml` `path` (the template builds to `dist/index.js`).2. From the **repo root**, commit the app directory — its `data_app.yaml`, the built bundle (the file `path` points at), the source, and the lockfile — and **push**:@@ -538,7 +544,7 @@ Data apps are delivered by Git, not uploaded — you commit the app directory an> **Don't offer to "deploy" the app or ask how the bundle reaches a staging environment** — there is no separate deploy step, and the question only confuses users: Metabase imports the committed bundle straight from the connected repo on its next sync. Once the change is on the branch Metabase syncs from — however the user gets it there (a merged PR, or a push straight to that branch) — just tell them to pull it in and open the app in Metabase at `/apps/<slug>`.- **To update:** commit a new build and pull again.-- **To remove:** delete the app's directory from the repo and push — the next sync removes it. You can't delete a repo-managed app from the UI; the *Remove* action on the Data apps admin page appears only after the repo is disconnected, to clear out apps left behind.+- **To remove:** delete the app's directory from the repo and push — the next sync removes it.## Common pitfallsdiff --git a/skills/metabase-data-app-setup/template/README.md b/skills/metabase-data-app-setup/template/README.mdindex d07d1c385ec..941b35212d8 100644--- a/skills/metabase-data-app-setup/template/README.md+++ b/skills/metabase-data-app-setup/template/README.md@@ -4,10 +4,10 @@ A Metabase **data app** — a single-bundle React app (built with the EmbeddingSDK) that Metabase renders inside an isolated, sandboxed iframe at`/apps/<slug>`.-Data apps are delivered through **Git, not uploaded**: this directory lives at+Data apps are delivered through **Git**: this directory lives at`data_apps/<slug>/` inside a repository connected to Metabase via remote sync.You commit the built bundle (`dist/index.js`), and on the next remote-sync-import Metabase materializes the app and serves it. There is no upload step.+import Metabase materializes the app and serves it.## Developdiff --git a/skills/metabase-data-app-setup/template/data_app.yaml b/skills/metabase-data-app-setup/template/data_app.yamlindex 43fb1b9b106..9fae16f682d 100644--- a/skills/metabase-data-app-setup/template/data_app.yaml+++ b/skills/metabase-data-app-setup/template/data_app.yaml@@ -2,6 +2,8 @@ version: 1name: Data App Template+slug: data-app-template+description: A summary of what this app doespath: ./dist/index.js@@ -12,3 +14,10 @@ path: ./dist/index.js# allowed_hosts:# - https://api.example.com# - https://*.internal.acme.com++entity_id: replace-entity-id-xxx++serdes/meta:+- model: DataApp+ id: replace-entity-id-xxx+ label: data-app-templatediff --git a/src/metabase/internal_stats/data_apps.clj b/src/metabase/internal_stats/data_apps.cljindex 458bf9b8499..ad38a434bac 100644--- a/src/metabase/internal_stats/data_apps.clj+++ b/src/metabase/internal_stats/data_apps.clj@@ -3,7 +3,6 @@[metabase.internal-stats.db :as internal-stats.db]))(defn data-app-stats- "How many data apps this instance actually serves: enabled and synced without- error (`sync_error IS NULL`), matching what the UI treats as an openable app."+ "How many data apps this instance actually serves: enabled ones that aren't drafts."[]{:data-app-count (internal-stats.db/enabled-data-app-count)})diff --git a/src/metabase/internal_stats/db.clj b/src/metabase/internal_stats/db.cljindex 2c6a14a12ef..dc50df1866c 100644--- a/src/metabase/internal_stats/db.clj+++ b/src/metabase/internal_stats/db.clj@@ -10,9 +10,9 @@[toucan2.core :as t2]))(mu/defn enabled-data-app-count- "The number of enabled data apps without a sync error."+ "The number of enabled data apps that aren't drafts."[]- (t2/count :data_app :enabled true :sync_error nil))+ (t2/count :data_app :enabled true :draft false))(mu/defn embedded-dashboard-count"The number of unarchived Dashboards with embedding enabled."diff --git a/src/metabase/models/db.clj b/src/metabase/models/db.cljindex ed625f5a540..170371c63b1 100644--- a/src/metabase/models/db.clj+++ b/src/metabase/models/db.clj@@ -60,6 +60,7 @@:model/DashboardCard :metabase.dashboards.schema/dashboard-card.update:model/DashboardCardSeries :metabase.dashboards.schema/dashboard-card-series.update:model/DashboardTab :metabase.dashboards.schema/dashboard-tab.update+ :model/DataApp :metabase-enterprise.data-apps.schema/data-app.update:model/Database :metabase.warehouses.schema/database.update:model/Dimension :metabase.warehouse-schema.schema/dimension.update:model/Document :metabase.documents.schema/document.updatediff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex c4277b713fe..0834dbd6c30 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -587,6 +587,14 @@[entity](:serdes/meta entity))+(defmulti resource-paths+ "Paths of the `:serdes/resources` stored next to an ingested entity's YAML file, relative to its directory."+ {:arglists '([ingested])}+ ingested-model)++(defmethod resource-paths :default [_]+ nil)+(defmulti load-find-local"Given a path, tries to look up any corresponding local entity.@@ -764,8 +772,10 @@[{:label (:name entity) :key (:entity_id entity)}]])))(defmulti storage-path- "Returns a vector of maps with `:label` and optional `:key` for each path segment.- `:label` is the human-readable name; `:key` is a deduplication identity (entity_id, name, or nil).+ "Returns a vector of maps with `:label` and optional `:key` and `:style` for each path segment.+ `:label` is the human-readable name; `:key` is a deduplication identity (entity_id, name, or nil); `:style` is+ `:name` (the default) for a label that is slugified into a file name, or `:slug` for a label that is already a slug+ safe to use as a file name as is.Dispatches on model name."{:arglists '([entity ctx])}(fn [entity _] (ingested-model entity)))diff --git a/test/metabase/internal_stats/data_apps_test.clj b/test/metabase/internal_stats/data_apps_test.cljindex e0e9505bf3e..9d946185d47 100644--- a/test/metabase/internal_stats/data_apps_test.clj+++ b/test/metabase/internal_stats/data_apps_test.clj@@ -3,6 +3,7 @@[clojure.test :refer [deftest is testing]][metabase.internal-stats.data-apps :as sut][metabase.test :as mt]+ [metabase.util :as u][toucan2.core :as t2]))(defn- data-app!@@ -11,6 +12,7 @@[& {:as extra}](let [app-name (mt/random-name)](t2/insert! :data_app (merge {:name app-name+ :entity_id (u/generate-nano-id):display_name "App":bundle_path "data_apps/app/index.js":created_at :%now@@ -31,8 +33,8 @@(testing "an app the admin disabled is out of service, so it doesn't count"(create! :enabled false)(is (= 2 (- (:data-app-count (sut/data-app-stats)) before))))- (testing "an app that failed to sync is not served, so it doesn't count even when enabled"- (create! :sync_error "boom")+ (testing "a draft is not served, so it doesn't count even when enabled"+ (create! :draft true)(is (= 2 (- (:data-app-count (sut/data-app-stats)) before)))))(finally(t2/delete! :data_app :name [:in @app-names])))))diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.cljindex c91bd6f7d22..6af3a2686ea 100644--- a/test/metabase/test/util.clj+++ b/test/metabase/test/util.clj@@ -179,6 +179,12 @@:name (u.random/random-name):position 0}))+ :model/DataApp+ (fn [_] {:entity_id (u/generate-nano-id)+ :name (u/lower-case-en (u.random/random-name))+ :display_name (u.random/random-name)+ :bundle_path "dist/index.js"})+:model/Database(fn [_] (default-timestamped{:details {}diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex 9d6ebe5bc1b..e6c3e1ca093 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -103,7 +103,8 @@(defn- table-details [table](-> (merge (mt/obj->json->obj (mt/object-defaults :model/Table))(select-keys table [:active :created_at :db_id :description :display_name :entity_type- :id :name :rows :schema :updated_at :visibility_type :initial_sync_status]))+ :id :name :rows :schema :updated_at :visibility_type :initial_sync_status+ :is_writable]))(update :entity_type #(when % (str "entity/" (name %))))(update :visibility_type #(when % (name %)))(update :schema str)))