Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Make data apps regular git sync entities (#83455)

Alexander Polyankina14a43b8f38c
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 46a8384e461..baa037bc04a 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1723,6 +1723,7 @@
:model/DashboardCard
:model/DashboardCardSeries
:model/DashboardTab
+ :model/DataApp
:model/Database
:model/Dimension
:model/Document
@@ -3736,10 +3737,11 @@
enterprise/data-apps
{:team "Embedding"
- :api #{metabase-enterprise.data-apps.api metabase-enterprise.data-apps.init metabase-enterprise.data-apps.sync}
+ :api #{metabase-enterprise.data-apps.api metabase-enterprise.data-apps.init}
:uses #{api
api-scope
collections
+ events
lib
lib.be
lib.schema
@@ -3757,7 +3759,8 @@
:model/Permissions
:model/PermissionsGroup
:model/Table
- :model/User}}
+ :model/User}
+ :model-exports #{:model/DataApp}}
enterprise/data-studio
{:team "UX West"
@@ -4193,7 +4196,6 @@
task.history
util
warehouse-schema-overlay
- enterprise/data-apps
enterprise/serialization
enterprise/transforms.python}
:model-exports #{:model/RemoteSyncTask}
@@ -4201,6 +4203,7 @@
:model/Card
:model/Collection
:model/Dashboard
+ :model/DataApp
:model/Document
:model/Field
:model/FieldUserSettings
diff --git a/e2e/embedding-sdk-host-apps/vite-6-data-app-host-app/data_app.yaml b/e2e/embedding-sdk-host-apps/vite-6-data-app-host-app/data_app.yaml
index 112f1b2c983..9a2498dc118 100644
--- a/e2e/embedding-sdk-host-apps/vite-6-data-app-host-app/data_app.yaml
+++ b/e2e/embedding-sdk-host-apps/vite-6-data-app-host-app/data_app.yaml
@@ -1,5 +1,11 @@
-name: Vite 6 Data App
version: 1
+name: Vite 6 Data App
+slug: vite-6-data-app-host-app
path: ./dist/index.js
allowed_hosts:
- https://allowed.data-app.test
+entity_id: qxpaPkU_WRE2ZQu0cmpqD
+serdes/meta:
+- model: DataApp
+ id: qxpaPkU_WRE2ZQu0cmpqD
+ label: vite-6-data-app-host-app
diff --git a/e2e/support/assets/data-apps/custom-error-component/data_app.yaml b/e2e/support/assets/data-apps/custom-error-component/data_app.yaml
index 1a604fccf95..6e534facce0 100644
--- a/e2e/support/assets/data-apps/custom-error-component/data_app.yaml
+++ b/e2e/support/assets/data-apps/custom-error-component/data_app.yaml
@@ -1,3 +1,9 @@
-name: Custom Error Component
version: 1
+name: Custom Error Component
+slug: custom-error-component
path: ./dist/index.js
+entity_id: uWb09GFf1aqUzlNsbhPyF
+serdes/meta:
+- model: DataApp
+ id: uWb09GFf1aqUzlNsbhPyF
+ label: custom-error-component
diff --git a/e2e/support/assets/data-apps/isolation/data_app.yaml b/e2e/support/assets/data-apps/isolation/data_app.yaml
index 96961791805..3ff1fd0cd67 100644
--- a/e2e/support/assets/data-apps/isolation/data_app.yaml
+++ b/e2e/support/assets/data-apps/isolation/data_app.yaml
@@ -1,3 +1,9 @@
-name: Isolation
version: 1
+name: Isolation
+slug: isolation
path: ./dist/index.js
+entity_id: nu2o-JSPcsIfu3FM0ek24
+serdes/meta:
+- model: DataApp
+ id: nu2o-JSPcsIfu3FM0ek24
+ label: isolation
diff --git a/e2e/support/assets/data-apps/kitchen-sink/data_app.yaml b/e2e/support/assets/data-apps/kitchen-sink/data_app.yaml
index 13a6b1d1769..b63b7271490 100644
--- a/e2e/support/assets/data-apps/kitchen-sink/data_app.yaml
+++ b/e2e/support/assets/data-apps/kitchen-sink/data_app.yaml
@@ -1,3 +1,9 @@
-name: Kitchen Sink
version: 1
+name: Kitchen Sink
+slug: kitchen-sink
path: ./dist/index.js
+entity_id: JyvrOkPvwWQAQCwUEd1bb
+serdes/meta:
+- model: DataApp
+ id: JyvrOkPvwWQAQCwUEd1bb
+ label: kitchen-sink
diff --git a/e2e/support/assets/data-apps/synced-actions-app/data_app.yaml b/e2e/support/assets/data-apps/synced-actions-app/data_app.yaml
index c0811b7f749..c8bf7f8c6da 100644
--- a/e2e/support/assets/data-apps/synced-actions-app/data_app.yaml
+++ b/e2e/support/assets/data-apps/synced-actions-app/data_app.yaml
@@ -1,3 +1,9 @@
-name: Synced Actions App
version: 1
+name: Synced Actions App
+slug: synced-actions-app
path: ./dist/index.js
+entity_id: nPu5r1EndcM_D8TQfLk9c
+serdes/meta:
+- model: DataApp
+ id: nPu5r1EndcM_D8TQfLk9c
+ label: synced-actions-app
diff --git a/e2e/support/assets/data-apps/synced-app/data_app.yaml b/e2e/support/assets/data-apps/synced-app/data_app.yaml
index 3ff729ed314..3e282a17794 100644
--- a/e2e/support/assets/data-apps/synced-app/data_app.yaml
+++ b/e2e/support/assets/data-apps/synced-app/data_app.yaml
@@ -1,3 +1,9 @@
-name: Synced App
version: 1
+name: Synced App
+slug: synced-app
path: ./dist/index.js
+entity_id: 4vxutiAe703DZgttXwORO
+serdes/meta:
+- model: DataApp
+ id: 4vxutiAe703DZgttXwORO
+ label: synced-app
diff --git a/e2e/support/assets/example_synced_data_apps/data_apps/bad-config/data_app.yaml b/e2e/support/assets/example_synced_data_apps/data_apps/bad-config/data_app.yaml
deleted file mode 100644
index 89a1e6b0e9d..00000000000
--- a/e2e/support/assets/example_synced_data_apps/data_apps/bad-config/data_app.yaml
+++ /dev/null
@@ -1 +0,0 @@
-name: [unterminated
diff --git a/e2e/support/assets/example_synced_data_apps/data_apps/bad-config/index.js b/e2e/support/assets/example_synced_data_apps/data_apps/bad-config/index.js
deleted file mode 100644
index 2d569610c95..00000000000
--- a/e2e/support/assets/example_synced_data_apps/data_apps/bad-config/index.js
+++ /dev/null
@@ -1,2 +0,0 @@
-// Present, but unreachable: its data_app.yaml can't be parsed, so the app is skipped.
-export default () => ({ component: null });
diff --git a/e2e/support/assets/example_synced_data_apps/data_apps/broken-bundle/data_app.yaml b/e2e/support/assets/example_synced_data_apps/data_apps/broken-bundle/data_app.yaml
deleted file mode 100644
index d95ac47d8d7..00000000000
--- a/e2e/support/assets/example_synced_data_apps/data_apps/broken-bundle/data_app.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-name: Broken Bundle
-version: 1
-# Points at a file that isn't in the repo — a valid config, an un-loadable app.
-path: ./missing.js
diff --git a/e2e/support/assets/example_synced_data_apps/data_apps/good/data_app.yaml b/e2e/support/assets/example_synced_data_apps/data_apps/good/data_app.yaml
index 8d241b3ad25..712c4c8a6c4 100644
--- a/e2e/support/assets/example_synced_data_apps/data_apps/good/data_app.yaml
+++ b/e2e/support/assets/example_synced_data_apps/data_apps/good/data_app.yaml
@@ -1,4 +1,10 @@
+version: 1
name: Good App
+slug: good
description: A well-formed app that syncs cleanly
-version: 1
path: ./index.js
+entity_id: Ioxf30LzIQCGwbCNtaG62
+serdes/meta:
+- model: DataApp
+ id: Ioxf30LzIQCGwbCNtaG62
+ label: good
diff --git a/e2e/support/assets/example_synced_data_apps/data_apps/no-config/index.js b/e2e/support/assets/example_synced_data_apps/data_apps/no-config/index.js
deleted file mode 100644
index 9c1c2183649..00000000000
--- a/e2e/support/assets/example_synced_data_apps/data_apps/no-config/index.js
+++ /dev/null
@@ -1,3 +0,0 @@
-// A bundle with no data_app.yaml beside it — nothing declares this directory an
-// app, so sync doesn't discover it.
-export default () => ({ component: null });
diff --git a/e2e/support/assets/example_synced_data_apps/data_apps/second-app/data_app.yaml b/e2e/support/assets/example_synced_data_apps/data_apps/second-app/data_app.yaml
new file mode 100644
index 00000000000..57d564db974
--- /dev/null
+++ b/e2e/support/assets/example_synced_data_apps/data_apps/second-app/data_app.yaml
@@ -0,0 +1,9 @@
+version: 1
+name: Second App
+slug: second-app
+path: ./index.js
+entity_id: A-0Jd9Jt80EIbt1hOavWS
+serdes/meta:
+- model: DataApp
+ id: A-0Jd9Jt80EIbt1hOavWS
+ label: second-app
diff --git a/e2e/support/assets/example_synced_data_apps/data_apps/second-app/index.js b/e2e/support/assets/example_synced_data_apps/data_apps/second-app/index.js
new file mode 100644
index 00000000000..e68ee3ca4c4
--- /dev/null
+++ b/e2e/support/assets/example_synced_data_apps/data_apps/second-app/index.js
@@ -0,0 +1,3 @@
+// A placeholder bundle — this fixture exercises sync/materialization + the admin
+// listing + the bundle endpoint, not rendering, so the bytes are served opaquely.
+export default () => ({ component: null });
diff --git a/e2e/support/helpers/e2e-data-app-helpers.ts b/e2e/support/helpers/e2e-data-app-helpers.ts
index f212c0f8819..8bcb1933a59 100644
--- a/e2e/support/helpers/e2e-data-app-helpers.ts
+++ b/e2e/support/helpers/e2e-data-app-helpers.ts
@@ -23,21 +23,20 @@ export const visitDataAppRoute = (route: string) => {
export const fakeDataApp = (overrides: Partial<DataApp> = {}): DataApp => ({
id: 1,
+ entity_id: "e2eFakeDataAppEntityI",
name: DATA_APP_NAME,
display_name: DATA_APP_DISPLAY_NAME,
description: null,
version: 1,
outdated: false,
- bundle_path: `data_apps/${DATA_APP_NAME}/dist/index.js`,
+ bundle_path: "dist/index.js",
enabled: true,
+ draft: false,
resource_collection_id: null,
permission_group_id: null,
table_ids: [],
allowed_hosts: [],
bundle_hash: "e2e-bundle-hash",
- last_synced_sha: "e2e0000",
- last_synced_at: "2024-01-01T00:00:00Z",
- sync_error: null,
created_at: "2024-01-01T00:00:00Z",
updated_at: "2024-01-01T00:00:00Z",
...overrides,
@@ -358,7 +357,7 @@ export function createDataAppApiKey() {
/**
* A second app beside the host app, for cases that need two of them. It reuses
* the host app's `node_modules`, so `defineQuery` still resolves through the
- * published SDK, and its directory name becomes the app's slug.
+ * published SDK, and its manifest declares `slug` as its slug.
*/
export function createSecondDataApp(slug: string) {
cy.task("scaffoldDataApp", { appName: slug, sdkFrom: dataAppHostAppRoot() });
diff --git a/e2e/support/helpers/e2e-data-app-tasks.ts b/e2e/support/helpers/e2e-data-app-tasks.ts
index 3e6ad76bec4..da800cc0f66 100644
--- a/e2e/support/helpers/e2e-data-app-tasks.ts
+++ b/e2e/support/helpers/e2e-data-app-tasks.ts
@@ -120,7 +120,7 @@ export async function scaffoldDataApp({
);
fs.writeFileSync(
path.join(root, "data_app.yaml"),
- `name: ${appName}\npath: ./dist/index.js\n`,
+ `name: ${appName}\nslug: ${appName}\npath: ./dist/index.js\n`,
);
return root;
diff --git a/e2e/test-host-app/data-apps/data-app-dev-diagnostics.cy.spec.ts b/e2e/test-host-app/data-apps/data-app-dev-diagnostics.cy.spec.ts
index f411c7719f2..ae16093dcf4 100644
--- a/e2e/test-host-app/data-apps/data-app-dev-diagnostics.cy.spec.ts
+++ b/e2e/test-host-app/data-apps/data-app-dev-diagnostics.cy.spec.ts
@@ -262,10 +262,12 @@ describe("Embedding SDK: data-app dev diagnostics", () => {
.findByText(/allowed_hosts changed since the dev server started/)
.should("not.exist");
- // `allowed_hosts` is the manifest's last key, so appending stays valid YAML.
cy.writeFile(
DATA_APP_DEV_MANIFEST_PATH,
- `${originalManifest} - https://added.example\n`,
+ originalManifest.replace(
+ " - https://allowed.data-app.test\n",
+ " - https://allowed.data-app.test\n - https://added.example\n",
+ ),
);
// No reload and no dev-server restart: the watcher re-validates and the
diff --git a/e2e/test-host-app/data-apps/sync-resources-actions.cy.spec.ts b/e2e/test-host-app/data-apps/sync-resources-actions.cy.spec.ts
index a99c4df0500..314808d2bab 100644
--- a/e2e/test-host-app/data-apps/sync-resources-actions.cy.spec.ts
+++ b/e2e/test-host-app/data-apps/sync-resources-actions.cy.spec.ts
@@ -17,7 +17,7 @@ const { H } = cy;
const TEST_TABLE = "scoreboard_actions";
const MODEL_NAME = "Scoreboard model";
-/** `syncResources` takes the app's slug from its directory name. */
+/** The `slug` the app's `data_app.yaml` declares. */
const APP_SLUG = "vite-6-data-app-host-app";
const APP_ROOT = () => dataAppHostAppRoot();
diff --git a/e2e/test-host-app/data-apps/sync-resources.cy.spec.ts b/e2e/test-host-app/data-apps/sync-resources.cy.spec.ts
index c7cf5639ff8..18179e1bf9b 100644
--- a/e2e/test-host-app/data-apps/sync-resources.cy.spec.ts
+++ b/e2e/test-host-app/data-apps/sync-resources.cy.spec.ts
@@ -18,7 +18,7 @@ import type { Card, DataApp } from "metabase-types/api";
const { H } = cy;
const { ORDERS_ID } = SAMPLE_DATABASE;
-/** `syncResources` takes the app's slug from its directory name. */
+/** The `slug` the app's `data_app.yaml` declares. */
const APP_SLUG = "vite-6-data-app-host-app";
/** The fields these specs read off a card the app collection holds. */
@@ -29,10 +29,17 @@ const LOCKFILE = () => `${APP_ROOT()}/resources_metadata.json`;
const QUERIES_FILE = () => `${APP_ROOT()}/queries/orders.query.ts`;
const MANIFEST_FILE = () => `${APP_ROOT()}/data_app.yaml`;
-const AUTHORED_MANIFEST = `name: Vite 6 Data App
+const AUTHORED_MANIFEST = `version: 1
+name: Vite 6 Data App
+slug: vite-6-data-app-host-app
path: ./dist/index.js
allowed_hosts:
- https://allowed.data-app.test
+entity_id: qxpaPkU_WRE2ZQu0cmpqD
+serdes/meta:
+- model: DataApp
+ id: qxpaPkU_WRE2ZQu0cmpqD
+ label: vite-6-data-app-host-app
`;
/**
diff --git a/e2e/test/scenarios/data-apps/sync-resources-production-actions.cy.spec.ts b/e2e/test/scenarios/data-apps/sync-resources-production-actions.cy.spec.ts
index d538c8c5ee2..3d57e6bd719 100644
--- a/e2e/test/scenarios/data-apps/sync-resources-production-actions.cy.spec.ts
+++ b/e2e/test/scenarios/data-apps/sync-resources-production-actions.cy.spec.ts
@@ -23,7 +23,18 @@ const MANIFEST_FILE = () => `${APP_ROOT()}/data_app.yaml`;
const LOCKFILE = () => `${APP_ROOT()}/resources_metadata.json`;
/** The manifest as source control holds it. */
-const AUTHORED_MANIFEST = `name: ${APP_DISPLAY_NAME}\npath: ./dist/index.js\n`;
+const AUTHORED_MANIFEST = [
+ "version: 1",
+ `name: ${APP_DISPLAY_NAME}`,
+ `slug: ${APP_SLUG}`,
+ "path: ./dist/index.js",
+ "entity_id: nPu5r1EndcM_D8TQfLk9c",
+ "serdes/meta:",
+ "- model: DataApp",
+ " id: nPu5r1EndcM_D8TQfLk9c",
+ " label: synced-actions-app",
+ "",
+].join("\n");
/** The declaration as source control holds it, with no generated ID yet. */
const declaration = (sourceActionId: number) =>
diff --git a/e2e/test/scenarios/data-apps/sync-resources-production.cy.spec.ts b/e2e/test/scenarios/data-apps/sync-resources-production.cy.spec.ts
index fd1d0decb94..b2d292e1386 100644
--- a/e2e/test/scenarios/data-apps/sync-resources-production.cy.spec.ts
+++ b/e2e/test/scenarios/data-apps/sync-resources-production.cy.spec.ts
@@ -22,7 +22,18 @@ const MANIFEST_FILE = () => `${APP_ROOT()}/data_app.yaml`;
const LOCKFILE = () => `${APP_ROOT()}/resources_metadata.json`;
/** The manifest as source control holds it. */
-const AUTHORED_MANIFEST = `name: ${APP_DISPLAY_NAME}\npath: ./dist/index.js\n`;
+const AUTHORED_MANIFEST = [
+ "version: 1",
+ `name: ${APP_DISPLAY_NAME}`,
+ `slug: ${APP_SLUG}`,
+ "path: ./dist/index.js",
+ "entity_id: 4vxutiAe703DZgttXwORO",
+ "serdes/meta:",
+ "- model: DataApp",
+ " id: 4vxutiAe703DZgttXwORO",
+ " label: synced-app",
+ "",
+].join("\n");
/** The declaration as source control holds it, with no generated ID yet. */
const AUTHORED_DECLARATION = [
diff --git a/e2e/test/scenarios/data-apps/sync.cy.spec.ts b/e2e/test/scenarios/data-apps/sync.cy.spec.ts
index bed4ffb67ee..325ba2a13b1 100644
--- a/e2e/test/scenarios/data-apps/sync.cy.spec.ts
+++ b/e2e/test/scenarios/data-apps/sync.cy.spec.ts
@@ -1,17 +1,8 @@
const { H } = cy;
/**
- * Drives a real remote-sync pull of a repo whose `data_apps/` covers every
- * materialization outcome, and asserts each is handled the way the backend
- * intends (see `data-apps.sync` / `data-apps.config`):
- *
- * good/ valid config + bundle -> materialized, served
- * broken-bundle/ valid config, missing bundle -> row with "Sync failed", not served
- * bad-config/ malformed data_app.yaml -> skipped, no row
- * no-config/ a bundle but no data_app.yaml -> not discovered, no row
- *
- * A bad app never blocks a good one, and neither a bad config nor a missing
- * bundle removes an app that isn't theirs.
+ * Drives a real remote-sync pull of a repo whose `data_apps/` holds two apps, each a `data_app.yaml` with its bundle
+ * next to it, and asserts that pulls materialize and remove them.
*/
describe("scenarios > data apps > repo sync", () => {
beforeEach(() => {
@@ -21,18 +12,15 @@ describe("scenarios > data apps > repo sync", () => {
H.setupGitSync();
});
- it("materializes each app per its config/bundle, isolating the broken ones", () => {
+ it("materializes each app with its bundle", () => {
H.copySyncedCollectionFixture();
H.copySyncedDataAppsFixture();
- H.commitToRepo("Add data apps with mixed config/bundle states");
+ H.commitToRepo("Add data apps");
H.configureGitAndPullChanges("read-write");
cy.visit("/admin/settings/apps");
cy.findByTestId("admin-layout-content").within(() => {
- // The good app is materialized and shown as synced. Scope to its own row so
- // its status can't be satisfied by another app's — a "Sync failed" leaking
- // onto the good app (or "Synced" onto the broken one) must fail the test.
cy.findByTestId("data-app-list-item-good")
.scrollIntoView()
.within(() => {
@@ -40,97 +28,37 @@ describe("scenarios > data apps > repo sync", () => {
cy.findByText("A well-formed app that syncs cleanly").should(
"be.visible",
);
- cy.findByText(/^Synced/).should("be.visible");
- });
-
- // The app whose bundle is missing still appears — with its failure, not hidden.
- // Its name is plain text, not a link: a sync-failed app can't be opened.
- cy.findByTestId("data-app-list-item-broken-bundle")
- .scrollIntoView()
- .within(() => {
- cy.findByText("Broken Bundle").should("be.visible");
- cy.findByRole("link", { name: "Broken Bundle" }).should("not.exist");
- cy.findByText("Sync failed").should("be.visible");
});
-
- // The malformed config and the config-less directory produced no app at all.
- cy.findByText("/apps/bad-config").should("not.exist");
- cy.findByText("/apps/no-config").should("not.exist");
+ cy.findByTestId("data-app-list-item-second-app").should("exist");
});
- // The API tells the same story: exactly the two apps, and only the good one serves a bundle.
cy.request("GET", "/api/apps").then(({ body: apps }) => {
expect(apps.map((app: { name: string }) => app.name).sort()).to.deep.eq([
- "broken-bundle",
"good",
+ "second-app",
]);
});
cy.request("/api/apps/good/bundle").its("status").should("eq", 200);
-
- cy.request({
- url: "/api/apps/broken-bundle/bundle",
- failOnStatusCode: false,
- }).then(({ status, body }) => {
- expect(status).to.eq(404);
- expect(body).to.deep.eq({ error: "Bundle not synced yet" });
- });
-
- for (const slug of ["bad-config", "no-config"]) {
- cy.request({
- url: `/api/apps/${slug}/bundle`,
- failOnStatusCode: false,
- }).then(({ status, body }) => {
- expect(status).to.eq(404);
- expect(body).to.eq("Not found.");
- });
- }
-
- // And what a user opening the broken app sees: its metadata loads (the app
- // exists), the host frames it, and the bundle 404 surfaces from inside the
- // iframe as the "isn't ready yet" screen — driven by the real pull, no mocks.
- // (The not-found screen for a missing app is covered in viewing.cy.spec.ts.)
- cy.visit("/apps/broken-bundle");
- H.main()
- .findByText(/isn.t ready yet/i, { timeout: 30000 })
- .should("be.visible");
});
- it("prunes an app whose directory is removed from the repo on the next sync", () => {
+ it("removes an app whose directory is removed from the repo on the next sync", () => {
H.copySyncedCollectionFixture();
H.copySyncedDataAppsFixture();
H.commitToRepo("Add data apps");
H.configureGitAndPullChanges("read-write");
- // Both apps are materialized from the first pull.
- cy.request("GET", "/api/apps").then(({ body: apps }) => {
- expect(apps.map((app: { name: string }) => app.name).sort()).to.deep.eq([
- "broken-bundle",
- "good",
- ]);
- });
-
- // Delete the good app's directory from the repo and sync again. The connected
- // repo is the source of truth, so the app must be pruned — not left serving.
cy.exec(`rm -rf -- "${H.LOCAL_GIT_PATH}/data_apps/good"`);
H.commitToRepo("Remove the good app from the repo");
H.configureGitAndPullChanges("read-write");
- // `good` is gone; `broken-bundle` (still in the repo) survives.
cy.request("GET", "/api/apps").then(({ body: apps }) => {
expect(apps.map((app: { name: string }) => app.name)).to.deep.eq([
- "broken-bundle",
+ "second-app",
]);
});
cy.request({ url: "/api/apps/good", failOnStatusCode: false })
.its("status")
.should("eq", 404);
-
- // The admin list reflects the removal.
- cy.visit("/admin/settings/apps");
- cy.findByTestId("admin-layout-content").within(() => {
- cy.findByTestId("data-app-list-item-broken-bundle").should("exist");
- cy.findByTestId("data-app-list-item-good").should("not.exist");
- });
});
});
diff --git a/e2e/test/scenarios/data-apps/user-access.cy.spec.ts b/e2e/test/scenarios/data-apps/user-access.cy.spec.ts
index d5feefe605f..ec032ee8c2b 100644
--- a/e2e/test/scenarios/data-apps/user-access.cy.spec.ts
+++ b/e2e/test/scenarios/data-apps/user-access.cy.spec.ts
@@ -352,10 +352,17 @@ describe("scenarios > data apps > user access (EMB-2328)", () => {
cy.task("writeDataAppFiles", {
files: {
[`${H.LOCAL_GIT_PATH}/data_apps/${SYNCED_APP_SLUG}/data_app.yaml`]: [
+ "version: 1",
"name: Good App",
+ `slug: ${SYNCED_APP_SLUG}`,
"path: ./index.js",
"allowed_hosts:",
` - ${ALLOWED_HOST}`,
+ "entity_id: Ioxf30LzIQCGwbCNtaG62",
+ "serdes/meta:",
+ "- model: DataApp",
+ " id: Ioxf30LzIQCGwbCNtaG62",
+ ` label: ${SYNCED_APP_SLUG}`,
"",
].join("\n"),
},
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/README.md b/enterprise/backend/src/metabase_enterprise/data_apps/README.md
index 00a8f97f5bd..ff1ab058675 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/README.md
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/README.md
@@ -1,125 +1,85 @@
# Data apps (backend)
A **data app** is a JS bundle, authored in a git repository, that Metabase serves and runs inside a
-Near Membrane sandbox in the browser. This README covers the backend half: how apps get from the
-repository into the app DB, and how they're served. The sandbox, the app runtime, and the host
+Near Membrane sandbox in the browser. This README covers the backend half: how apps are stored, created,
+serialized, and served. The sandbox, the app runtime, and the host
iframe live on the frontend and are out of scope here.
-## Where apps come from
+## What an app is
-Apps are **not uploaded**. A single repository is connected to Metabase through remote-sync
-(Admin → Settings → Remote sync), and each app lives in its own directory under `data_apps/` at the
-repo root:
+`:model/DataApp` is a regular serdes entity. A row holds the app's manifest fields — `name` (the
+slug, served at `/apps/<slug>`), `display_name`, `description`, the data app contract `version`, the
+`bundle_path`, and the `allowed_hosts` its sandboxed bundle may `fetch`/XHR — plus the bundle bytes
+themselves, cached in the app DB so serving never reads a repository.
-```
-data_apps/
- sales/ # the directory name is the slug: /apps/sales
- data_app.yaml
- dist/index.js # the built bundle, committed
- ops/
- data_app.yaml
- dist/app.js
-```
+The model's hooks hold its invariants: every write is normalized and validated against the column
+schemas in `schema.clj`, `bundle_hash` always matches `bundle`, an inserted app gets the collection
+and permission group it owns (see Permissions), and a deleted one loses them. Its default fields
+leave the bundle out, so listing apps never drags bundles out of the DB; `db.clj`'s
+`data-app-bundle` reads it explicitly.
-`data_app.yaml` declares the display name and the bundle path relative to the app's own directory,
-plus three optional fields: a one-line `description` shown beside the name in the admin UI, the
-`version` of the data app contract the app was built for (1 when absent; an app below the version
-this Metabase serves is outdated — badged for admins, hidden from everyone else, and a 409 to open),
-and the origins the sandboxed bundle may `fetch`/XHR. See `config.clj` for the format and its
-validation.
+An app with a lower `version` than this Metabase serves is outdated — badged for admins, hidden
+from everyone else, and a 409 to open.
-**The directory name is the slug.** Nothing in the config declares it. This is what makes slug
-collisions structurally impossible — a repo can't hold two `data_apps/sales` directories, and
-discovery takes at most one config per directory.
+## Serialization
-## The sync pipeline
-
-Data apps have no sync of their own; they ride the remote-sync import. Every pull — the manual
-"Pull changes" button, the auto-import poll, or startup — ends in
-`remote-sync.impl/materialize-data-apps!`, which adapts the imported snapshot to plain reader fns
-and calls `data-apps.sync/sync-from-snapshot!`.
+A serialized app is a `data_app.yaml` in its own directory under `data_apps/`, with its bundle as a
+plain file next to it at its `path`:
```
-remote-sync import → snapshot → sync-from-snapshot! → discover configs
- → materialize each app
- → prune apps absent from the repo
+data_apps/
+ sales/
+ data_app.yaml # serdes/meta, entity_id, slug, name, description, version, path, allowed_hosts
+ dist/index.js # the bundle
```
-`data_apps/` is deliberately **not** a serdes path, so apps are invisible to the serialization
-layer and are counted separately in the pull summary (see `fold-data-app-changes` in
-`remote_sync/impl.clj` — without it, a pull whose only change is a data app reports "no changes").
-
-Apps are synced independently. Each app's row and its resource links commit together, and pruning
-runs in its own transaction. A database or resource setup error rolls back that app's update, then
-records `sync_error` separately.
-
-## The repository is the source of truth
-
-A sync **upserts every app it finds and deletes every row whose directory is gone.** The
-consequences are worth stating explicitly, because they're the questions that come up:
-
-| Event | What happens to apps |
-| ----------------------------------- | ------------------------------------------------------------------------- |
-| App directory removed from the repo | Row and cached bundle deleted on the next sync |
-| Repo switched to a different one | Previous repo's apps are absent from the new snapshot, so they're dropped |
-| Repo unlinked | Nothing — unlinking runs no sync, so apps survive |
-| Repo has no `data_apps/` at all | All apps removed |
-| Failed clone/fetch | Nothing — that throws before a snapshot exists, so deletion never fires |
-
-Pruning is by **directory presence**, not by successful parse. An app whose directory is still
-there but whose `data_app.yaml` is momentarily broken keeps its row and its last-good bundle, and is
-marked with a `sync_error` — see failure isolation below. Local `enabled` state does not protect a
-row: an app disabled in the admin UI and then deleted from the repo is still removed. Draft rows —
-created via `POST /api/apps/:slug/draft` ahead of an app's first import — are the one exemption:
-pruning skips them until a sync materializes the app and clears the flag.
-
-## Failure isolation
-
-A sync must never take a working app offline, and one bad app must never abort the others. So
-failures are recorded per app rather than thrown:
+The YAML keeps the keys a hand-written manifest uses: `slug` is the `name` column, `name` the
+`display_name`, and `path` the `bundle_path`. The bundle travels as a serdes *resource file*: the
+entity carries it in `:serdes/resources` on export, the storage writers put it next to the YAML,
+and ingestion reads the paths `serdes/resource-paths` returns back in. A resource path must stay
+inside the entity's directory.
-- **Malformed `data_app.yaml`** — collected into `:config-errors`. An app that already has a row is
- marked with `sync_error` (so the UI shows it as failed rather than silently presenting the
- last-good bundle as freshly synced); an app that has no row yet is simply not materialized.
-- **Bundle missing or over the size cap** — the row's metadata is still upserted with `sync_error`
- set, and the previously cached bundle is kept.
-- **Anything thrown at all** — `sync-from-snapshot!` catches and logs it, returning nil, so a
- data-app failure can't break the surrounding remote-sync import.
+Only the manifest is serialized. `enabled` is admin-owned and never leaves the instance; the
+collection, permission group, and `table_ids` are server-managed; `bundle_hash` is recomputed from
+the bundle on import. Drafts are not exported.
-Because the bundle is cached in the app DB rather than read from the repo per request, a broken sync
-degrades to "the app still serves its last good bundle, and the admin sees why it's stale."
+An import matches an app by `entity_id`, falling back to its slug so it takes over a draft, and
+reasserts the app's resources. It is a no-op without the `:data-apps` feature.
-## What survives a sync
+Remote sync treats data apps like any other entity, globally rather than per collection. Because
+the bundle is a separate file, a pull that changes only a bundle, or an export that touches an app,
+takes the full rather than the incremental path. An app's directory also holds its source, which
+serialization doesn't own, so exports replace only the YAML and resource files in `data_apps/`.
-`enabled` is admin-owned and is never written by a sync — an app disabled in the admin UI stays
-disabled across pulls, and new rows get the DB default of true.
+## Drafts
-`:changed` accounting deliberately ignores `last_synced_sha` / `last_synced_at`: re-syncing
-identical content at a new commit is not a change, so the pull summary reports real edits only.
+`POST /api/apps/:slug/draft` reserves a slug and creates the app's resources before the app itself
+exists, so its resources can be prepared ahead of time. Creating the app fills the draft.
## Serving
Routes are mounted at `/api/apps` (`api.clj`). Not `/app/*` — the server reserves that for static
assets (`metabase.server.routes/static-files-handler`).
-- `GET /api/apps` — list; `?available=true` filters to enabled apps with no sync error.
+- `GET /api/apps` — list; `?available=true` filters to enabled apps that aren't drafts.
- `GET /api/apps/:slug` — metadata for one enabled app.
- `GET /api/apps/:slug/bundle` — the cached bytes, with a content-hash ETag and `If-None-Match`
→ 304. Carries `X-Metabase-Data-App-Allowed-Hosts`, which the iframe reads to configure its
sandbox fetch allowlist.
- `GET /api/apps/sandbox-host` — the empty document loaded as the Near-Membrane realm iframe,
carrying the CSP that confines `'unsafe-eval'` to that realm.
-- `PUT /api/apps/:slug` — toggle `enabled` (superuser).
+- `POST /api/apps` — create an app from its manifest fields and bundle text, filling a draft with
+ the same slug (superuser).
+- `PUT /api/apps/:slug` — update manifest fields or the bundle, or toggle `enabled` (superuser).
- `DELETE /api/apps/:slug` — drop a row, its bundle, and its owned resources (superuser).
-- `POST /api/apps/:slug/draft` — create or reuse a draft row with its resources before the app's
- first import (superuser).
+- `POST /api/apps/:slug/draft` — create or reuse a draft row with its resources (superuser).
- `POST /api/apps/:slug/query` — resolve an authored query definition into a serializable
Metabase query plus the table IDs it touches (superuser).
- `GET /api/apps/repo-status` — whether a repo is connected (superuser).
Responses are field-filtered by role: superusers get full metadata, everyone else gets `name` and
`display_name` only. The bundle blob is never serialized into JSON, and metadata reads go through
-`data-apps.db`'s non-blob helpers so listing apps doesn't drag the bundles out of the DB.
+the model's default fields, which leave the bundle out, so listing apps doesn't drag the bundles out of the DB.
`csp.clj` exposes an app's `allowed_hosts` to the core security middleware through a `defenterprise`
hook, which drives the `connect-src` of the iframe document's CSP. It's a separate namespace so the
@@ -127,8 +87,8 @@ middleware's lookup doesn't pull in route code.
## Permissions
-Each app owns two server-managed resources (`resources.clj`), created on draft or first import and
-reasserted on every sync: a **collection** holding the copies the app is served from (saved
+Each app owns two server-managed resources (`resources.clj`), created with the app (or its draft)
+and reasserted on every import: a **collection** holding the copies the app is served from (saved
questions, action models, table-sourced metrics) and a **permissions group** its users belong to.
The group is set database-level `view-data :blocked` on every database, so it grants **no data
@@ -153,10 +113,11 @@ status.
| Namespace | Responsibility |
| --------------------- | --------------------------------------------------------------------------------------------------- |
-| `sync.clj` | Discovery, materialization, pruning, drafts. The entry point remote-sync calls. |
-| `config.clj` | `data_app.yaml` parsing and validation; the `data_apps/` layout constants. |
+| `apps.clj` | Creating apps and drafts; the connected repository's URL. |
+| `config.clj` | The serialized layout and data app contract version constants. |
+| `schema.clj` | Column schemas, with the normalization and validation every write goes through. |
| `api.clj` | The `/api/apps` endpoints, bundle serving, ETag handling. |
| `resources.clj` | Lifecycle of the app-owned collection and permission group: creation, view-data blocking, deletion. |
-| `models/data_app.clj` | The `:model/DataApp` Toucan model, permissions, blob coercion. |
+| `models/data_app.clj` | The `:model/DataApp` Toucan model: hooks, permissions, default fields, serialization. |
| `csp.clj` | `allowed_hosts` lookup for the core CSP middleware. |
| `init.clj` | Loads the above so endpoints, models, and hooks register. |
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/api.clj b/enterprise/backend/src/metabase_enterprise/data_apps/api.clj
index 54c3ddb94d1..f7c449afefd 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/api.clj
@@ -1,22 +1,24 @@
(ns metabase-enterprise.data-apps.api
- "Data-app endpoints, mounted at `/api/apps`. Serves the bundles materialized by
- [[metabase-enterprise.data-apps.sync]]; see `README.md` in this directory for
- where apps come from and how the pieces fit together.
+ "Data-app endpoints, mounted at `/api/apps`: create, read, update, and delete data apps, and serve their
+ bundles. See `README.md` in this directory for how the pieces fit together.
Note: we can't use `/app/...` for the frontend or `/api/app/...` for the API
because Metabase's server reserves `/app/*` for serving static assets (see
`metabase.server.routes/static-files-handler`)."
(:require
[clojure.string :as str]
+ [metabase-enterprise.data-apps.apps :as data-apps.apps]
[metabase-enterprise.data-apps.config :as data-app.config]
[metabase-enterprise.data-apps.db :as data-apps.db]
+ [metabase-enterprise.data-apps.models.data-app :as data-app]
[metabase-enterprise.data-apps.query-definition :as query-definition]
- [metabase-enterprise.data-apps.sync :as data-app.sync]
+ [metabase-enterprise.data-apps.schema :as data-apps.schema]
[metabase-enterprise.data-apps.user-access :as data-app.user-access]
[metabase.api-scope.data-app :as api-scope]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]
+ [metabase.events.core :as events]
[metabase.lib-be.core :as lib-be]
[metabase.lib-be.schema :as lib-be.schema]
[metabase.lib.core :as lib]
@@ -69,7 +71,7 @@
origin (conj origin))))
(defn- repo-status []
- (let [url (data-app.sync/repo-url)]
+ (let [url (data-apps.apps/repo-url)]
{:configured (some? url)
:url url}))
@@ -99,12 +101,29 @@
[:table_ids [:sequential ms/PositiveInt]]
[:has_user_permission_warnings {:optional true} :boolean]
[:bundle_hash [:maybe :string]]
- [:last_synced_sha [:maybe :string]]
- [:last_synced_at [:maybe :any]]
- [:sync_error [:maybe :string]]
[:created_at :any]
[:updated_at :any]])
+(def ^:private CreateDataAppRequest
+ [:map {:closed true}
+ [:name ::data-apps.schema/slug]
+ [:display_name ::data-apps.schema/display-name]
+ [:description {:optional true} ::data-apps.schema/description]
+ [:version {:optional true} ::data-apps.schema/version]
+ [:bundle_path ::data-apps.schema/bundle-path]
+ [:allowed_hosts {:optional true} ::data-apps.schema/allowed-hosts]
+ [:bundle ::data-apps.schema/bundle-text]])
+
+(def ^:private UpdateDataAppRequest
+ [:map {:closed true}
+ [:enabled {:optional true} :boolean]
+ [:display_name {:optional true} ::data-apps.schema/display-name]
+ [:description {:optional true} ::data-apps.schema/description]
+ [:version {:optional true} ::data-apps.schema/version]
+ [:bundle_path {:optional true} ::data-apps.schema/bundle-path]
+ [:allowed_hosts {:optional true} ::data-apps.schema/allowed-hosts]
+ [:bundle {:optional true} ::data-apps.schema/bundle-text]])
+
(def ^:private PublicDataAppResponse
[:map {:closed true}
[:name ms/NonBlankString]
@@ -253,12 +272,11 @@
app)
(api.macros/defendpoint :get "/" :- [:sequential [:or DataAppResponse PublicDataAppResponse]]
- "List the data apps provided by the connected repository. Pass `available=true`
- to return only enabled apps without sync errors. An outdated app is never
+ "List the data apps. Pass `available=true` to return only enabled apps that aren't drafts. An outdated app is never
available, and otherwise listed only to superusers, who see it badged."
[_route-params
{:keys [available]} :- [:map {:closed true} [:available {:optional true} [:maybe :boolean]]]]
- (let [apps (->> (data-apps.db/non-blob-data-apps available)
+ (let [apps (->> (data-apps.db/data-apps available)
(remove #(and (or available (not api/*is-superuser?*))
(data-app.config/outdated? %)))
(mapv api/read-check))
@@ -266,28 +284,51 @@
(data-app.user-access/groups-with-permission-warnings apps))]
(mapv (partial data-app-list-response warning-group-ids) apps)))
+;; NOTE on the `slug-regex` constraint: the default path-param matcher allows
+;; slashes inside a segment, so `/:slug` would otherwise swallow `/x/bundle`.
+;; The regex also excludes the literal `repo-status` sub-route above.
+(defn- with-bundle
+ "The DataApp `changes` with their `bundle` text as the bytes the app is served."
+ [changes]
+ (cond-> changes
+ (contains? changes :bundle) (update :bundle data-app/file->bundle)))
+
+(defn- write-check-data-app
+ "The DataApp named `slug` without its bundle, checked for existence and write permissions."
+ [slug]
+ (api/write-check (api/check-404 (data-apps.db/data-app-by-slug slug))))
+
+(api.macros/defendpoint :post "/" :- DataAppResponse
+ "Create a data app from its manifest fields and bundle. A draft with the same slug becomes the app."
+ [_route-params
+ _query-params
+ body :- CreateDataAppRequest]
+ (api/create-check :model/DataApp body)
+ (let [app (data-apps.db/data-app (data-apps.apps/create-app! (with-bundle body)))]
+ (events/publish-event! :event/data-app-create {:object app :user-id api/*current-user-id*})
+ (data-app-response app)))
+
;; NOTE on the `slug-regex` constraint: the default path-param matcher allows
;; slashes inside a segment, so `/:slug` would otherwise swallow `/x/bundle`.
;; The regex also excludes the literal `repo-status` sub-route above.
(api.macros/defendpoint :put ["/:slug" :slug slug-regex] :- DataAppResponse
- "Enable or disable a single data app. Disabled apps are not served."
+ "Update a data app's manifest fields or bundle, or enable or disable it. Disabled apps are not served."
[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]
_query-params
- {:keys [enabled]} :- [:map {:closed true} [:enabled :boolean]]]
- (api/check-superuser)
- (let [app (api/check-404 (data-apps.db/non-blob-data-app-by-slug slug))]
- (data-apps.db/update-data-app! (:id app) {:enabled enabled})
- (data-app-response (data-apps.db/non-blob-data-app (:id app)))))
+ changes :- UpdateDataAppRequest]
+ (let [app (write-check-data-app slug)]
+ (when (seq changes)
+ (data-apps.db/update-data-app! (:id app) (with-bundle changes)))
+ (let [app (data-apps.db/data-app (:id app))]
+ (events/publish-event! :event/data-app-update {:object app :user-id api/*current-user-id*})
+ (data-app-response app))))
(api.macros/defendpoint :delete ["/:slug" :slug slug-regex] :- :nil
- "Remove a single data app (its row and cached bundle). Intended for clearing out
- apps left behind by a repository that is no longer connected: while a repo is
- connected, an app still in it is re-materialized by the next sync, and one no
- longer in it is pruned by that sync anyway."
+ "Delete a data app, its bundle, and the collection and permission group it owns."
[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]]
- (api/check-superuser)
- ;; The delete returns the row count; a 0 means the slug wasn't there → 404.
- (api/check-404 (pos? (data-apps.db/delete-data-app-by-slug! slug)))
+ (let [app (write-check-data-app slug)]
+ (data-apps.db/delete-data-app! (:id app))
+ (events/publish-event! :event/data-app-delete {:object app :user-id api/*current-user-id*}))
;; a `nil` body is rendered as a 204; matches the `:- :nil` response schema
;; above (returning `generic-204-no-content` would fail that validation).
nil)
@@ -297,14 +338,13 @@
[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]
_query-params
{table-ids :table_ids} :- TableDependenciesRequest]
- (api/check-superuser)
- (let [app (api/check-404 (data-apps.db/non-blob-data-app-by-slug slug))
+ (let [app (write-check-data-app slug)
table-ids (vec (sort table-ids))]
(api/check-400 (= (set table-ids)
(data-apps.db/existing-table-ids table-ids))
(tru "One or more tables do not exist."))
(data-apps.db/update-data-app! (:id app) {:table_ids table-ids})
- (data-app-response (data-apps.db/non-blob-data-app (:id app)))))
+ (data-app-response (data-apps.db/data-app (:id app)))))
(api.macros/defendpoint :post ["/:slug/user-permission-warnings" :slug slug-regex]
:- [:sequential PermissionWarning]
@@ -312,8 +352,7 @@
[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]
_query-params
{user-ids :user_ids} :- PermissionWarningsRequest]
- (api/check-superuser)
- (let [app (api/check-404 (data-apps.db/non-blob-data-app-by-slug slug))
+ (let [app (write-check-data-app slug)
users (data-apps.db/users-for-permission-warnings user-ids)]
(api/check-404 (= (count users) (count user-ids)))
(api/check-400 (every? :is_active users)
@@ -363,8 +402,7 @@
[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]
_query-params
query-def :- ::query-definition/query-definition]
- (api/check-superuser)
- (api/check-404 (data-apps.db/non-blob-data-app-by-slug slug))
+ (write-check-data-app slug)
(let [{source-type :type, table-id :id} (get-in query-def [:stages 0 :source])
_ (api/check-400 (= (keyword source-type) :table)
"Data app query definitions must use a table source.")
@@ -385,8 +423,7 @@
[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]
_query-params
{dataset-queries :dataset_queries} :- QueryTableDependenciesRequest]
- (api/check-superuser)
- (api/check-404 (data-apps.db/non-blob-data-app-by-slug slug))
+ (write-check-data-app slug)
{:table_ids (->> dataset-queries
(mapcat (fn [dataset-query]
(-> (lib-be/application-database-metadata-provider (:database dataset-query))
@@ -397,13 +434,11 @@
vec)})
(api.macros/defendpoint :post ["/:slug/draft" :slug slug-regex] :- DataAppResponse
- "Create or reuse a data app draft before its first repository import."
- [{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]]
- (api/check-superuser)
- (api/check-400 (data-app.config/valid-slug? slug)
- "Data app draft slugs must use lowercase letters, numbers, and dashes.")
- (data-app.sync/ensure-draft! slug)
- (data-app-response (data-apps.db/non-blob-data-app-by-slug slug)))
+ "Create or reuse a data app draft, which reserves a slug and the app's resources before the app is created."
+ [{:keys [slug]} :- [:map {:closed true} [:slug ::data-apps.schema/slug]]]
+ (api/create-check :model/DataApp {:name slug})
+ (data-apps.apps/ensure-draft! slug)
+ (data-app-response (data-apps.db/data-app-by-slug slug)))
;; Not tagged `data-apps:base`, though the bundle route below is — which looks backwards until
;; you place the two callers. `DataAppView` fetches this metadata on the *host* page to decide
@@ -412,7 +447,7 @@
(api.macros/defendpoint :get ["/:slug" :slug slug-regex] :- [:or DataAppResponse PublicDataAppResponse]
"Fetch metadata for a single enabled data app by its slug."
[{:keys [slug]} :- [:map {:closed true} [:slug ms/NonBlankString]]]
- (let [app (read-check-data-app (data-apps.db/enabled-non-blob-data-app-by-slug slug))]
+ (let [app (read-check-data-app (data-apps.db/enabled-data-app-by-slug slug))]
(data-app-response (cond-> app (not api/*is-superuser?*) check-not-outdated))))
(api.macros/defendpoint :get ["/:slug/bundle" :slug slug-regex] :- :any
@@ -426,7 +461,7 @@
respond
raise]
(try
- (let [row (check-not-outdated (read-check-data-app (data-apps.db/enabled-non-blob-data-app-by-slug slug)))
+ (let [row (check-not-outdated (read-check-data-app (data-apps.db/enabled-data-app-by-slug slug)))
hash (:bundle_hash row)
etag (some->> hash (format "\"%s\""))]
(cond
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/apps.clj b/enterprise/backend/src/metabase_enterprise/data_apps/apps.clj
new file mode 100644
index 00000000000..02f6bc155e7
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/apps.clj
@@ -0,0 +1,63 @@
+(ns metabase-enterprise.data-apps.apps
+ "Creating data apps and their drafts, and the connected repository's URL."
+ (:require
+ [clojure.string :as str]
+ [metabase-enterprise.data-apps.db :as data-apps.db]
+ [metabase-enterprise.data-apps.resources :as data-app.resources]
+ [metabase.api.common :as api]
+ [metabase.settings.core :as setting]
+ [metabase.util.i18n :refer [tru]]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private draft-bundle-path
+ "The bundle path of a draft, until the app is created."
+ "dist/index.js")
+
+(defn repo-url
+ "The connected remote-sync repository URL, or nil when none is configured. Read
+ by keyword so this OSS namespace has no compile-time dependency on the
+ enterprise remote-sync module; returns nil when that module (and its setting)
+ isn't loaded."
+ []
+ (let [url (try (setting/get :remote-sync-url) (catch Throwable _ nil))]
+ (when-not (str/blank? url)
+ url)))
+
+(defn- create-draft!
+ [slug]
+ (let [app (t2/with-transaction [_conn]
+ (when-not (data-apps.db/data-app-exists? slug)
+ (data-apps.db/insert-data-app!
+ {:name slug
+ :display_name slug
+ :bundle_path draft-bundle-path
+ :draft true}))
+ (data-apps.db/data-app-by-slug slug))]
+ (data-app.resources/ensure-resources! app)))
+
+(defn ensure-draft!
+ "Create a data app draft when needed and ensure its permission resources.
+ Creating the app later fills the same row with its manifest and bundle."
+ [slug]
+ (try
+ (create-draft! slug)
+ (catch Throwable e
+ (if (data-apps.db/data-app-exists? slug)
+ (create-draft! slug)
+ (throw e)))))
+
+(defn create-app!
+ "Create the data app `row`, filling the draft with its slug if there is one, and return its ID. Throws a 409
+ when a data app that isn't a draft already has the slug."
+ [{slug :name :as row}]
+ (t2/with-transaction [_conn]
+ (let [existing (data-apps.db/data-app-by-slug slug)
+ row (assoc row :draft false)]
+ (api/check (or (nil? existing) (:draft existing))
+ [409 (tru "A data app with this slug already exists.")])
+ (if existing
+ (do (data-apps.db/update-data-app! (:id existing) (dissoc row :name))
+ (:id existing))
+ (data-apps.db/insert-data-app-returning-pk! row)))))
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/config.clj b/enterprise/backend/src/metabase_enterprise/data_apps/config.clj
index b549647f49b..38fa86b1889 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/config.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/config.clj
@@ -1,29 +1,19 @@
(ns metabase-enterprise.data-apps.config
- "Parsing + validation of a per-app `data_app.yaml`, which sits in the app's own
- directory under `data_apps/` (see `README.md` in this directory for the layout):
+ "Constants of a data app's manifest, which a serialized app keeps in the `data_app.yaml` in its own directory
+ under `data_apps/` (see `README.md` in this directory for the layout); the columns' schemas are in
+ [[metabase-enterprise.data-apps.schema]]:
- name: Sales dashboard # display name
+ slug: sales # the slug the app is served at (column `name`)
+ name: Sales dashboard # display name (column `display_name`)
description: Pipeline … # optional — one line on what the app does
version: 1 # optional — data app contract version, 1 when absent
- path: dist/index.js # bundle path, relative to this app's directory
+ path: dist/index.js # bundle path, relative to this app's directory (column `bundle_path`)
allowed_hosts: # optional — origins the sandboxed bundle may fetch/XHR
- https://api.example.com
- - https://*.internal.acme.com"
- (:require
- [clojure.java.io :as io]
- [clojure.string :as str]
- [metabase.util :as u]
- [metabase.util.i18n :refer [tru]]
- [metabase.util.yaml :as yaml])
- (:import
- (java.io ByteArrayInputStream)))
+ - https://*.internal.acme.com")
(set! *warn-on-reflection* true)
-(def config-file-name
- "Name of the per-app config file (used in error messages)."
- "data_app.yaml")
-
(def apps-dir
"Directory at the repo root that holds one subdirectory per data app."
"data_apps")
@@ -34,180 +24,7 @@
listed only to admins, badged, and never opened."
1)
-(def ^:private initial-app-version
- "The `version` of an app whose manifest declares none: every such app predates the
- field, so this stays 1 while [[supported-app-version]] moves on."
- 1)
-
(defn outdated?
"Whether `app` was built for a contract version older than [[supported-app-version]]."
[{:keys [version]}]
(< version supported-app-version))
-
-(defn- parse-version
- "Validate the optional `version`: a positive whole number, [[initial-app-version]]
- when absent. Throws a 400 for anything else rather than coercing it, since a
- string or a decimal here is a typo, not a version."
- [raw ^String dir]
- (cond
- (nil? raw)
- initial-app-version
-
- (and (integer? raw) (pos? raw))
- (long raw)
-
- :else
- (throw (ex-info (tru "{0}/{1}: \"version\" must be a positive whole number, such as 1." dir config-file-name)
- {:status-code 400}))))
-
-(def ^:private slug-pattern
- "Lowercase letters/numbers separated by single dashes. An app *directory* must
- match this: the name is used verbatim as the slug, never normalized. Folding
- case here would reintroduce the collisions the directory naming rules out —
- git happily holds both `data_apps/Sales` and `data_apps/sales`."
- #"[a-z0-9]+(?:-[a-z0-9]+)*")
-
-(def ^:private reserved-slugs
- "Slugs that collide with literal `/api/apps/*` sub-routes (see the API's
- `slug-regex`). An app with one of these would sync but be unreachable, so we
- reject it during parsing."
- #{"repo-status" "sandbox-host"})
-
-(defn- normalize-path
- "Trim and drop a leading `./` so the path is relative to the app directory."
- [p]
- (-> (str p) str/trim (str/replace #"^\./" "")))
-
-(defn- normalize-description
- "Trim and fold whitespace runs, so a YAML block scalar still stores as one line."
- [d]
- (-> (str d) str/trim (str/replace #"\s+" " ")))
-
-(def ^:private max-description-chars
- "Cap on `description`. Checked here, not by the column type, so an over-long one fails just its own app."
- 255)
-
-(defn- parse-description
- "Validate + normalize the optional one-line `description`. Returns nil when it's
- absent or blank; throws a 400 when it's present but not a string — a YAML list or
- mapping would otherwise `str`-ify into Clojure collection syntax and sync as-is —
- or when it exceeds [[max-description-chars]]."
- [raw ^String dir]
- (when (some? raw)
- (when-not (string? raw)
- (throw (ex-info (tru "{0}/{1}: \"description\" must be a one-line string." dir config-file-name)
- {:status-code 400})))
- (let [d (not-empty (normalize-description raw))]
- (when (and d (> (count d) max-description-chars))
- (throw (ex-info (tru "{0}/{1}: \"description\" is a one-line summary — it must be {2} characters or fewer."
- dir config-file-name max-description-chars)
- {:status-code 400})))
- d)))
-
-(def ^:private allowed-host-re
- "A single `allowed_hosts` entry: an origin the app's sandboxed bundle may
- `fetch`/XHR — scheme + host, an optional `*.` subdomain wildcard, and an
- optional port. No path/query/`*`/non-http(s) scheme: it names an
- origin, e.g. `https://api.example.com` or `https://*.internal.acme.com`."
- #"(?i)https?://(\*\.)?[a-z0-9-]+(\.[a-z0-9-]+)*(:\d+)?")
-
-(defn- normalized-allowed-host
- "Trim, lowercase, and drop trailing slashes from one `allowed_hosts` entry."
- [entry]
- (some-> entry str str/trim u/lower-case-en (str/replace #"/+$" "")))
-
-(defn- parse-allowed-host
- "Normalize + validate a single `allowed_hosts` entry (see [[allowed-host-re]]).
- Throws an `ex-info` with `:status-code` 400 when it isn't an origin-only value."
- [entry ^String dir]
- (let [s (normalized-allowed-host entry)]
- (when-not (and (seq s) (re-matches allowed-host-re s))
- (throw (ex-info (tru "{0}/{1}: \"{2}\" is not a valid allowed_hosts entry — use an origin like https://api.example.com or https://*.example.com."
- dir config-file-name (str entry))
- {:status-code 400})))
- s))
-
-(defn- parse-allowed-hosts
- "Validate + normalize the optional `allowed_hosts` list from a parsed config.
- Each entry must be an origin-only value (see [[allowed-host-re]]); returns a
- lowercased, de-duplicated vector, or `[]` when the key is absent. Throws an
- `ex-info` with `:status-code` 400 on a non-list value or an invalid entry."
- [parsed ^String dir]
- (let [raw (:allowed_hosts parsed)]
- (cond
- (nil? raw) []
- (not (sequential? raw))
- (throw (ex-info (tru "{0}/{1}: \"allowed_hosts\" must be a list." dir config-file-name)
- {:status-code 400}))
- :else
- (->> raw
- (map #(parse-allowed-host % dir))
- distinct
- vec))))
-
-(defn- path-traversal? [path]
- (some #(= ".." %) (str/split path #"/")))
-
-(defn- parse-yaml [^bytes bytes ^String dir]
- (try
- (with-open [r (io/reader (ByteArrayInputStream. bytes) :encoding "UTF-8")]
- (yaml/parse-stream r))
- (catch Exception e
- (throw (ex-info (tru "Could not parse {0}/{1}: {2}" dir config-file-name (ex-message e))
- {:status-code 400})))))
-
-(defn dir-slug
- "The app's slug: the name of its directory (`data_apps/sales` -> `sales`). Used
- verbatim, never normalized. Public so discovery can label an app by its
- directory even when its `data_app.yaml` fails to parse (so a transiently broken
- config isn't treated as a removal)."
- [^String dir]
- (subs dir (inc (str/last-index-of dir "/"))))
-
-(defn valid-slug?
- "Whether `slug` can name a data app directory and API route."
- [slug]
- (and (string? slug)
- (re-matches slug-pattern slug)
- (not (contains? reserved-slugs slug))))
-
-(defn parse-app-config
- "Parse the bytes of one `data_app.yaml` from the app directory `dir` (e.g.
- `data_apps/sales`) into `{:slug ..., :display_name ..., :description ...,
- :version ..., :path ..., :allowed_hosts [...]}`. The slug is the directory's
- name; `path` is relative to the directory; `:description` is an optional
- one-liner (nil when absent or blank, capped at [[max-description-chars]]);
- `:version` is the contract version the app was built for (1 when absent);
- `:allowed_hosts` is a (possibly empty) vector of origins the sandboxed bundle
- may reach. Throws an
- `ex-info` with `:status-code` 400 on malformed or incomplete content — including
- a directory whose name isn't a usable slug, since that app has no URL to be served at."
- [^bytes bytes ^String dir]
- (let [parsed (parse-yaml bytes dir)
- slug (dir-slug dir)
- name (some-> (:name parsed) str str/trim not-empty)
- description (parse-description (:description parsed) dir)
- version (parse-version (:version parsed) dir)
- path (some-> (:path parsed) normalize-path not-empty)
- allowed-hosts (parse-allowed-hosts parsed dir)]
- (when-not (re-matches slug-pattern slug)
- (throw (ex-info (tru "{0}: the app directory''s name is its slug, so it must be lowercase letters, numbers, and dashes." dir)
- {:status-code 400})))
- (when (contains? reserved-slugs slug)
- (throw (ex-info (tru "{0}: \"{1}\" is a reserved slug — rename the directory." dir slug)
- {:status-code 400})))
- (when-not name
- (throw (ex-info (tru "{0}/{1}: \"name\" is required." dir config-file-name)
- {:status-code 400})))
- (when-not path
- (throw (ex-info (tru "{0}/{1}: \"path\" is required." dir config-file-name)
- {:status-code 400})))
- (when (path-traversal? path)
- (throw (ex-info (tru "{0}/{1}: \"path\" must not contain \"..\"." dir config-file-name)
- {:status-code 400})))
- {:slug slug
- :display_name name
- :description description
- :version version
- :path path
- :allowed_hosts allowed-hosts}))
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/csp.clj b/enterprise/backend/src/metabase_enterprise/data_apps/csp.clj
index fb49d3dd750..ec8f7de99c1 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/csp.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/csp.clj
@@ -12,5 +12,5 @@
CSP `connect-src` so the sandboxed bundle can fetch/XHR those origins."
:feature :data-apps
[slug]
- (or (:allowed_hosts (data-apps.db/enabled-non-blob-data-app-by-slug slug))
+ (or (:allowed_hosts (data-apps.db/enabled-data-app-by-slug slug))
[]))
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj b/enterprise/backend/src/metabase_enterprise/data_apps/db.clj
index 21dfb1feaf3..9e76e4387ea 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/db.clj
@@ -9,47 +9,59 @@
[metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[toucan2.core :as t2]))
-(def ^:private non-blob-columns
- "Columns to select for normal data-app metadata reads, excluding the raw bundle blob."
- [:model/DataApp :id :name :display_name :description :version :bundle_path :enabled :allowed_hosts
- :resource_collection_id :permission_group_id :table_ids :draft
- :bundle_hash :last_synced_sha :last_synced_at :sync_error
- :created_at :updated_at])
-
-(mu/defn non-blob-data-app
+(def non-blob-columns
+ "Every DataApp column except the raw bundle blob."
+ [:id :entity_id :name :display_name :description :version :bundle_path :enabled :allowed_hosts
+ :resource_collection_id :permission_group_id :table_ids :draft :bundle_hash :created_at :updated_at])
+
+(def ^:private non-blob-model
+ (into [:model/DataApp] non-blob-columns))
+
+(mu/defn data-app
"The DataApp with `data-app-id` without its bundle, or nil."
[data-app-id :- ms/PositiveInt]
- (t2/select-one non-blob-columns :id data-app-id))
+ (t2/select-one non-blob-model :id data-app-id))
-(mu/defn non-blob-data-app-by-slug
+(mu/defn data-app-by-slug
"The DataApp named `slug` without its bundle, or nil."
[slug :- :string]
- (t2/select-one non-blob-columns :name slug))
+ (t2/select-one non-blob-model :name slug))
+
+(mu/defn draft-by-slug
+ "The draft DataApp named `slug` without its bundle, or nil."
+ [slug :- :string]
+ (t2/select-one non-blob-model :name slug :draft true))
-(mu/defn enabled-non-blob-data-app-by-slug
+(mu/defn enabled-data-app-by-slug
"The enabled DataApp named `slug` without its bundle, or nil."
[slug :- :string]
- (t2/select-one non-blob-columns :name slug :enabled true))
+ (t2/select-one non-blob-model :name slug :enabled true))
-(mu/defn non-blob-data-apps
- "Every DataApp without its bundle, ordered by display name; only the enabled, error-free ones when `available?`."
+(mu/defn data-apps
+ "Every DataApp without its bundle, ordered by display name; only the enabled ones that aren't drafts when
+ `available?`."
[available? :- [:maybe :boolean]]
- (t2/select non-blob-columns
+ (t2/select non-blob-model
(cond-> {:order-by [[:display_name :asc]]}
available? (assoc :where [:and
[:= :enabled true]
- [:= :sync_error nil]]))))
+ [:= :draft false]]))))
(mu/defn data-app-bundle
"The bundle bytes of the DataApp with `data-app-id`."
[data-app-id :- ms/PositiveInt]
- (t2/select-one-fn :bundle :model/DataApp :id data-app-id))
-
-(mu/defn data-apps-sync-info
- "The sync-relevant columns of every DataApp."
- []
- (t2/select [:model/DataApp :name :display_name :description :version :allowed_hosts :bundle_path
- :bundle_hash :sync_error]))
+ (t2/select-one-fn :bundle [:model/DataApp :bundle] :id data-app-id))
+
+(mu/defn reducible-data-apps-with-bundles
+ "A reducible of the DataApp rows, bundles included, whose `filter-column` is one of `filter-ids` (every row when
+ `filter-column` is nil), ordered ascending by `order-columns`."
+ [filter-column :- [:maybe :keyword]
+ filter-ids :- [:maybe [:sequential [:maybe [:or :int :string]]]]
+ order-columns :- [:maybe [:sequential :keyword]]]
+ (t2/reducible-select [:model/DataApp :*]
+ (cond-> {}
+ filter-column (assoc :where [:in filter-column filter-ids])
+ (seq order-columns) (assoc :order-by (mapv (fn [column] [column :asc]) order-columns)))))
(defn table-database-id
"The database ID for the Table with `table-id`, or nil."
@@ -87,37 +99,21 @@
[row :- ::data-apps.schema/data-app.update]
(t2/insert! :model/DataApp row))
+(mu/defn insert-data-app-returning-pk! :- ms/PositiveInt
+ "Insert the DataApp `row`, returning its ID."
+ [row :- ::data-apps.schema/data-app.update]
+ (t2/insert-returning-pk! :model/DataApp row))
+
(mu/defn update-data-app!
"Apply `changes` to the DataApp with `data-app-id`."
[data-app-id :- ms/PositiveInt
changes :- ::data-apps.schema/data-app.update]
(t2/update! :model/DataApp :id data-app-id changes))
-(mu/defn update-data-app-by-slug!
- "Apply `changes` to the DataApp named `slug`."
- [slug :- :string
- changes :- ::data-apps.schema/data-app.update]
- (t2/update! :model/DataApp :name slug changes))
-
-(mu/defn delete-data-app-by-slug!
- "Delete the DataApp named `slug`, returning the number deleted."
- [slug :- :string]
- (t2/delete! :model/DataApp :name slug))
-
-(mu/defn delete-data-apps-not-named!
- "Delete non-draft DataApps whose name is not one of `slugs`, returning the number deleted."
- [slugs :- [:set :string]]
- (t2/delete! :model/DataApp :name [:not-in slugs] :draft false))
-
-(mu/defn delete-all-data-apps!
- "Delete every non-draft DataApp, returning the number deleted."
- []
- (t2/delete! :model/DataApp :draft false))
-
-(defn publish-data-app-drafts!
- "Mark drafts named by `slugs` as published."
- [slugs]
- (t2/update! :model/DataApp :name [:in slugs] :draft true {:draft false}))
+(mu/defn delete-data-app!
+ "Delete the DataApp with `data-app-id`."
+ [data-app-id :- ms/PositiveInt]
+ (t2/delete! :model/DataApp :id data-app-id))
(defn permission-group
"The permission group with `group-id`, or nil."
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/init.clj b/enterprise/backend/src/metabase_enterprise/data_apps/init.clj
index 53251438552..2a02e6ec974 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/init.clj
@@ -1,7 +1,7 @@
(ns metabase-enterprise.data-apps.init
(:require
[metabase-enterprise.data-apps.api]
+ [metabase-enterprise.data-apps.apps]
[metabase-enterprise.data-apps.config]
[metabase-enterprise.data-apps.csp]
- [metabase-enterprise.data-apps.models.data-app]
- [metabase-enterprise.data-apps.sync]))
+ [metabase-enterprise.data-apps.models.data-app]))
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj b/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj
index d7b9017dd37..5f89bbe249b 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj
@@ -1,14 +1,25 @@
(ns metabase-enterprise.data-apps.models.data-app
(:require
+ [metabase-enterprise.data-apps.config :as data-app.config]
[metabase-enterprise.data-apps.db :as data-apps.db]
[metabase-enterprise.data-apps.resources :as data-app.resources]
+ [metabase-enterprise.data-apps.schema :as data-apps.schema]
[metabase.api.common :as api]
+ [metabase.events.core :as events]
+ [metabase.lib.core :as lib]
[metabase.models.interface :as mi]
+ [metabase.models.serialization :as serdes]
[metabase.premium-features.core :refer [defenterprise]]
+ [metabase.util.i18n :refer [tru]]
+ [metabase.util.malli :as mu]
[methodical.core :as methodical]
- [toucan2.core :as t2])
+ [toucan2.core :as t2]
+ [toucan2.tools.default-fields :as t2.default-fields])
(:import
- (java.sql Blob)))
+ (java.nio.charset StandardCharsets)
+ (java.security MessageDigest)
+ (java.sql Blob)
+ (org.apache.commons.codec.binary Hex)))
(set! *warn-on-reflection* true)
@@ -33,25 +44,41 @@
(doto :model/DataApp
(derive :metabase/model)
+ (derive :hook/entity-id)
(derive :hook/timestamped?))
-(defn- default-last-synced-at
- "A sync that records `last_synced_sha` happened now unless it says otherwise."
+(t2.default-fields/define-default-fields :model/DataApp
+ data-apps.db/non-blob-columns)
+
+(events/derive! ::event :metabase/event)
+(doseq [e [:event/data-app-create :event/data-app-update :event/data-app-delete]]
+ (events/derive! e ::event))
+
+(defn- bytes-hash ^String [^bytes b]
+ (let [^MessageDigest md (MessageDigest/getInstance "SHA-256")]
+ (Hex/encodeHexString ^bytes (.digest md b))))
+
+(defn- with-bundle-hash
+ "The `row` with `bundle_hash` matching the `bundle` it sets."
[row]
(cond-> row
- (and (contains? row :last_synced_sha) (not (contains? row :last_synced_at)))
- (assoc :last_synced_at (mi/now))))
+ (contains? row :bundle) (assoc :bundle_hash (some-> ^bytes (:bundle row) bytes-hash))))
+
+(defn- prepare
+ "The DataApp `row` normalized and validated against `schema`, with the hash of any bundle it sets."
+ [schema row]
+ (->> row
+ (lib/normalize schema)
+ (mu/validate-throw schema)
+ with-bundle-hash))
(t2/define-before-insert :model/DataApp
[data-app]
- (default-last-synced-at data-app))
+ (prepare ::data-apps.schema/data-app.insert data-app))
(t2/define-before-update :model/DataApp
[data-app]
- (let [changes (t2/changes data-app)]
- (cond-> data-app
- (and (contains? changes :last_synced_sha) (not (contains? changes :last_synced_at)))
- (assoc :last_synced_at (mi/now)))))
+ (merge data-app (some->> (t2/changes data-app) (prepare ::data-apps.schema/data-app.update))))
;; Reads always see `allowed_hosts` as a vector, never nil — a row synced before
;; the column existed has NULL until it's re-synced. Guard on `contains?` so
@@ -78,6 +105,10 @@
[_model _instance]
api/*is-superuser?*)
+(t2/define-after-insert :model/DataApp
+ [app]
+ (merge app (data-app.resources/ensure-resources! app)))
+
(t2/define-before-delete :model/DataApp
[app]
(data-app.resources/delete-resources! app))
@@ -87,6 +118,70 @@
[data-app json-generator]
(next-method (dissoc data-app :bundle) json-generator))
+(defn- bundle->file
+ "The `bundle` bytes as the text of its file."
+ [^bytes bundle]
+ (String. bundle StandardCharsets/UTF_8))
+
+(defn file->bundle
+ "The `content` of a bundle file as the bytes the app is served."
+ ^bytes [^String content]
+ (.getBytes content StandardCharsets/UTF_8))
+
+(defn- ingested-bundle-path
+ "The normalized bundle path of the ingested data app `ingested`."
+ [ingested]
+ (lib/normalize ::data-apps.schema/bundle-path (:path ingested)))
+
+(defmethod serdes/make-spec "DataApp"
+ [_model-name _opts]
+ {:copy [:entity_id :description :version :allowed_hosts]
+ :skip [;; admin-owned state of this instance
+ :enabled
+ ;; set by the import itself
+ :draft :bundle_hash
+ ;; server-managed resources, recreated on import
+ :resource_collection_id :permission_group_id :table_ids]
+ :transform {:created_at (serdes/date)
+ :name {:as :slug :export identity :import identity}
+ :display_name {:as :name :export identity :import identity}
+ :bundle_path {:as :path :export identity :import identity}
+ :bundle {:as :serdes/resources
+ :export-with-context (fn [app _k bundle]
+ (if bundle
+ {(:bundle_path app) (bundle->file bundle)}
+ ::serdes/skip))
+ :import-with-context (fn [ingested _k files]
+ (let [path (ingested-bundle-path ingested)]
+ (file->bundle
+ (or (get files path)
+ (throw (ex-info (tru "Bundle file \"{0}\" not found." path)
+ {:status-code 400}))))))}}
+ :defaults {:description nil :version 1 :allowed_hosts []}})
+
+(defmethod serdes/extract-query "DataApp"
+ [model-name {:keys [filter-column filter-ids] :as opts}]
+ (eduction (remove :draft)
+ (data-apps.db/reducible-data-apps-with-bundles filter-column filter-ids
+ (serdes/extract-order-columns model-name opts))))
+
+(defmethod serdes/storage-path "DataApp" [app _ctx]
+ [{:label data-app.config/apps-dir}
+ {:label (:slug app) :key (:entity_id app) :style :slug}
+ {:label "data_app"}])
+
+(defmethod serdes/resource-paths "DataApp" [ingested]
+ [(ingested-bundle-path ingested)])
+
+(defmethod serdes/load-one! "DataApp"
+ [ingested maybe-local]
+ (let [local (or maybe-local (data-apps.db/draft-by-slug (:slug ingested)))
+ app (serdes/default-load-one! ingested local)]
+ (data-apps.db/update-data-app! (:id app) {:draft false})
+ (when local
+ (data-app.resources/ensure-resources! app))
+ app))
+
(defenterprise data-app-group-ids
"The data-app permission groups (those flagged `is_data_app_group`). SSO group sync must never touch
their membership."
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/resources.clj b/enterprise/backend/src/metabase_enterprise/data_apps/resources.clj
index 43e10fb3656..5c653c5206f 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/resources.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/resources.clj
@@ -76,7 +76,7 @@
"Create or restore the server-owned permission resources for `app` and return their IDs."
[app]
(perms/with-global-permissions-lock
- (let [app (data-apps.db/non-blob-data-app (:id app))
+ (let [app (data-apps.db/data-app (:id app))
group (permission-group! app)
collection (resource-collection! app)]
(data-apps.db/update-permission-group! (:id group)
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj b/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj
index bd27ba3c16a..ff0c1ccff90 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj
@@ -1,9 +1,95 @@
(ns metabase-enterprise.data-apps.schema
"Malli schemas for the data-apps module."
(:require
+ [clojure.string :as str]
+ [metabase.util :as u]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]))
+(def max-bundle-bytes
+ "Cap on a data app bundle's size (10 MiB)."
+ (* 10 1024 1024))
+
+(def ^:private reserved-slugs
+ "Slugs that collide with literal `/api/apps/*` sub-routes, so an app with one would be unreachable."
+ #{"repo-status" "sandbox-host"})
+
+(defn- decoders
+ "Schema properties decoding a value with `f` both in [[metabase.lib.core/normalize]] and at the API boundary."
+ [f]
+ {:decode/normalize f, :decode/api f})
+
+(defn- trim [x]
+ (cond-> x (string? x) str/trim))
+
+(defn- one-line [x]
+ (cond-> x (string? x) (-> (str/replace #"\s+" " ") str/trim not-empty)))
+
+(defn- relative-path [x]
+ (cond-> x (string? x) (-> str/trim (str/replace #"^\./" ""))))
+
+(defn- origin [x]
+ (cond-> x (string? x) (-> str/trim u/lower-case-en (str/replace #"/+$" ""))))
+
+(defn- version-or-1 [x]
+ (if (nil? x) 1 x))
+
+(defn- distinct-vec [x]
+ (cond-> x (nil? x) vec, (sequential? x) (-> distinct vec)))
+
+(mr/def ::slug
+ "A data app's slug: lowercase letters and numbers separated by single dashes, at most 100 characters."
+ [:and
+ {:error/message "must be lowercase letters, numbers, and dashes"}
+ [:string {:max 100}]
+ [:re #"\A[a-z0-9]+(?:-[a-z0-9]+)*\z"]
+ [:fn {:error/message "must not be a reserved slug"} (complement reserved-slugs)]])
+
+(mr/def ::display-name
+ "A data app's display name."
+ [:string (merge {:min 1} (decoders trim))])
+
+(mr/def ::description
+ "A data app's optional one-line summary, nil when blank."
+ [:maybe (decoders one-line) [:string {:max 255}]])
+
+(mr/def ::version
+ "The data app contract version an app was built for, 1 when it declares none."
+ [:int (merge {:min 1} (decoders version-or-1))])
+
+(mr/def ::bundle-path
+ "The path of a data app's bundle, relative to the app's directory and inside it."
+ [:and
+ (decoders relative-path)
+ ms/NonBlankString
+ [:fn {:error/message "must be relative to the app's directory and stay inside it"}
+ #(and (not (str/starts-with? % "/"))
+ (not (str/includes? % "\\"))
+ (not-any? #{"" "." ".."} (str/split % #"/" -1)))]
+ [:fn {:error/message "must not be the app's data_app.yaml"} #(not= "data_app.yaml" %)]])
+
+(mr/def ::allowed-host
+ "An origin a data app's sandboxed bundle may fetch: scheme, host with an optional `*.` subdomain wildcard, and an
+ optional port."
+ [:re
+ (merge {:error/message "must be an origin like https://api.example.com or https://*.example.com"}
+ (decoders origin))
+ #"\Ahttps?://(\*\.)?[a-z0-9-]+(\.[a-z0-9-]+)*(:\d+)?\z"])
+
+(mr/def ::allowed-hosts
+ "The origins a data app's sandboxed bundle may fetch, `[]` when nil."
+ [:sequential (decoders {:leave distinct-vec}) ::allowed-host])
+
+(mr/def ::bundle
+ "A data app's bundle bytes."
+ [:and
+ bytes?
+ [:fn {:error/message "must be less than 10 MiB"} #(<= (alength ^bytes %) max-bundle-bytes)]])
+
+(mr/def ::bundle-text
+ "The text of a data app's bundle file."
+ [:string {:max max-bundle-bytes}])
+
(mr/def ::data-app
"A DataApp as selected from the app DB: every column of `:data_app`."
[:merge
@@ -14,21 +100,28 @@
(mr/def ::data-app.update
"What an update (or insert) of a DataApp accepts: every column of `:data_app` except `id`, all optional."
[:map {:closed true}
- [:name {:optional true} [:maybe :string]]
- [:display_name {:optional true} [:maybe :string]]
- [:allowed_hosts {:optional true} [:maybe [:sequential :string]]]
- [:bundle_path {:optional true} [:maybe :string]]
- [:bundle {:optional true} [:maybe [:or bytes? :string]]]
+ [:entity_id {:optional true} [:maybe :string]]
+ [:name {:optional true} ::slug]
+ [:display_name {:optional true} ::display-name]
+ [:allowed_hosts {:optional true} ::allowed-hosts]
+ [:bundle_path {:optional true} ::bundle-path]
+ [:bundle {:optional true} [:maybe ::bundle]]
[:bundle_hash {:optional true} [:maybe :string]]
- [:last_synced_sha {:optional true} [:maybe :string]]
- [:last_synced_at {:optional true} [:maybe ms/TemporalInstant]]
- [:sync_error {:optional true} [:maybe :string]]
[:enabled {:optional true} [:maybe :boolean]]
[:created_at {:optional true} [:maybe ms/TemporalInstant]]
[:updated_at {:optional true} [:maybe ms/TemporalInstant]]
- [:description {:optional true} [:maybe :string]]
- [:version {:optional true} ms/PositiveInt]
+ [:description {:optional true} ::description]
+ [:version {:optional true} ::version]
[:resource_collection_id {:optional true} [:maybe ms/PositiveInt]]
[:permission_group_id {:optional true} [:maybe ms/PositiveInt]]
[:table_ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]
[:draft {:optional true} [:maybe :boolean]]])
+
+(mr/def ::data-app.insert
+ "What an insert of a DataApp accepts: [[::data-app.update]] with its slug, display name, and bundle path required."
+ [:merge
+ ::data-app.update
+ [:map
+ [:name ::slug]
+ [:display_name ::display-name]
+ [:bundle_path ::bundle-path]]])
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/sync.clj b/enterprise/backend/src/metabase_enterprise/data_apps/sync.clj
deleted file mode 100644
index 05855ef571b..00000000000
--- a/enterprise/backend/src/metabase_enterprise/data_apps/sync.clj
+++ /dev/null
@@ -1,296 +0,0 @@
-(ns metabase-enterprise.data-apps.sync
- "Materialize data apps from a synced repository snapshot — discovery, upsert, and
- pruning. [[sync-from-snapshot!]] is the entry point remote-sync calls on every
- import.
-
- See `README.md` in this directory for the pipeline, the source-of-truth rules
- (what a sync deletes, and what unlinking or switching repos does), and the
- failure-isolation guarantees this namespace implements.
-
- This namespace does no Git access of its own — the snapshot's `read-file` /
- `list-dir` come from remote-sync. The cached `bundle` blob is what serving
- reads, so a failed sync never takes a working app offline, and the admin
- `enabled` toggle is preserved across syncs."
- (:require
- [clojure.string :as str]
- [metabase-enterprise.data-apps.config :as data-app.config]
- [metabase-enterprise.data-apps.db :as data-apps.db]
- [metabase-enterprise.data-apps.resources :as data-app.resources]
- [metabase.permissions.core :as perms]
- [metabase.premium-features.core :as premium-features]
- [metabase.settings.core :as setting]
- [metabase.util.i18n :refer [tru]]
- [metabase.util.log :as log]
- [toucan2.core :as t2])
- (:import
- (java.security MessageDigest)
- (org.apache.commons.codec.binary Hex)))
-
-(set! *warn-on-reflection* true)
-
-(def ^:const max-bundle-bytes
- "Cap on a synced bundle's size (10 MiB)."
- (* 10 1024 1024))
-
-(defn- bytes-hash ^String [^bytes b]
- (let [^MessageDigest md (MessageDigest/getInstance "SHA-256")]
- (Hex/encodeHexString ^bytes (.digest md b))))
-
-(defn- ->bytes ^bytes [^String s]
- (.getBytes s "UTF-8"))
-
-(defn repo-url
- "The connected remote-sync repository URL, or nil when none is configured. Read
- by keyword so this OSS namespace has no compile-time dependency on the
- enterprise remote-sync module; returns nil when that module (and its setting)
- isn't loaded."
- []
- (let [url (try (setting/get :remote-sync-url) (catch Throwable _ nil))]
- (when-not (str/blank? url)
- url)))
-
-(defn- create-draft!
- [slug]
- (let [app (t2/with-transaction [_conn]
- (when-not (data-apps.db/data-app-exists? slug)
- (data-apps.db/insert-data-app!
- {:name slug
- :display_name slug
- :bundle_path (format "%s/%s/%s" data-app.config/apps-dir slug data-app.config/config-file-name)
- :sync_error (tru "Bundle not synced yet.")
- :draft true}))
- (data-apps.db/non-blob-data-app-by-slug slug))]
- (data-app.resources/ensure-resources! app)))
-
-(defn ensure-draft!
- "Create a data app draft when needed and ensure its permission resources.
- A later repository import fills the same row with the authoritative manifest
- and bundle."
- [slug]
- (try
- (create-draft! slug)
- (catch Throwable e
- (if (data-apps.db/data-app-exists? slug)
- (create-draft! slug)
- (throw e)))))
-
-;;; ----------------------------------------------------- Discovery -----------------------------------------------------
-
-(defn- discover-app-configs
- "Given the snapshot's `list-dir` and `read-file` fns (where `list-dir` returns a
- directory's immediate children as repo-root relative paths, and `read-file`
- returns file text or nil), iterate the folders under `data_apps/` and return one
- entry per folder that is an app — it holds a `data_app.yaml`; a folder without
- one is skipped. Each entry is a parsed app `{:slug :display_name :description
- :version :bundle :allowed_hosts}` (with `:bundle` the repo-root relative bundle path) or
- `{:config-error <message>}`. Parse/read failures are isolated per app so one bad
- config can't abort the sync.
-
- Reads only `data_apps/` and its app folders, never the whole tree. That matters
- beyond this fn: `data_apps/` is not a serdes path, so a pull that changes only
- data apps builds no ingestable at all (see remote-sync's
- `incremental-import-plan`) — discovery here is the only thing that would touch
- the tree, and it now stays proportional to the number of apps rather than to
- the size of the repo."
- [list-dir read-file]
- (for [dir (list-dir data-app.config/apps-dir)
- :let [config-path (str dir "/" data-app.config/config-file-name)]
- ;; a plain file under `data_apps/` lists no children, so it drops out here
- :when (some #{config-path} (list-dir dir))]
- (try
- (if-let [content (read-file config-path)]
- (let [{:keys [slug display_name description version path allowed_hosts]}
- (data-app.config/parse-app-config (->bytes content) dir)]
- {:slug slug
- :display_name display_name
- :description description
- :version version
- :bundle (str dir "/" path)
- :allowed_hosts allowed_hosts})
- {:slug (data-app.config/dir-slug dir), :config-error (tru "Could not read {0}." config-path)})
- (catch Throwable e
- {:slug (data-app.config/dir-slug dir), :config-error (ex-message e)}))))
-
-;;; ----------------------------------------------------- Materialize -----------------------------------------------------
-
-(defn- upsert-by-name!
- "Insert or update by slug. Never writes `:enabled`, so the admin toggle (and the
- DB default of true for new rows) is preserved across syncs."
- [slug row]
- (if (data-apps.db/data-app-exists? slug)
- (data-apps.db/update-data-app-by-slug! slug row)
- (data-apps.db/insert-data-app! (assoc row :name slug))))
-
-(def ^:private synced-metadata-keys
- "The manifest-derived columns a sync stores whether or not the bundle loaded."
- [:display_name :description :version :allowed_hosts :bundle_path])
-
-(defn- app-metadata-changed?
- "Whether the metadata a sync stores whether or not the bundle loaded differs from
- the `existing` row. The failure path stores exactly these, so it asks this
- directly; the success path adds the bundle to the question."
- [existing fields]
- (let [normalize (fn [row]
- (-> (select-keys row synced-metadata-keys)
- (update :allowed_hosts #(vec (or % [])))))]
- (not= (normalize existing) (normalize fields))))
-
-(defn- app-content-changed?
- "Whether the just-synced content differs from the `existing` row (nil = a new
- app). Compares only content-bearing fields — a `last_synced_sha` /
- `last_synced_at` bump on an otherwise-identical app is NOT a change, so callers
- can count real changes (e.g. for the remote-sync pull summary)."
- [existing {:keys [bundle_hash] :as fields}]
- (or (nil? existing)
- (some? (:sync_error existing))
- (not= (:bundle_hash existing) bundle_hash)
- (app-metadata-changed? existing fields)))
-
-(defn- mark-config-error!
- "Record a `data_app.yaml` parse failure on an app that already has a row, keeping the
- row and its cached bundle and setting `sync_error`. An app with no row yet isn't
- materialized at all — there's nothing to serve. Returns true when this changed the
- app's recorded state, so callers can count it like any other change."
- [existing slug message]
- (boolean
- (when (and existing (not= (:sync_error existing) message))
- (data-apps.db/update-data-app-by-slug! slug {:sync_error message})
- true)))
-
-(defn- prepare-app
- "Read one app's bundle before taking the permissions lock.
- A bundle failure preserves the cached bundle and prepares metadata with `sync_error` instead."
- [existing {:keys [slug display_name description version bundle sha read-file allowed_hosts]}]
- (let [metadata {:display_name display_name
- :description description
- :version version
- :allowed_hosts allowed_hosts
- :bundle_path bundle}]
- (try
- (let [content (read-file bundle)
- _ (when-not content
- (throw (ex-info (tru "Bundle file \"{0}\" not found in the repository." bundle)
- {:status-code 400})))
- ^bytes bytes (->bytes content)]
- (when (> (alength bytes) max-bundle-bytes)
- (throw (ex-info (tru "Bundle for \"{0}\" must be less than {1} MiB."
- slug (quot max-bundle-bytes (* 1024 1024)))
- {:status-code 413})))
- (let [fields (assoc metadata :bundle_hash (bytes-hash bytes))]
- {:fields (assoc fields :bundle bytes :last_synced_sha sha
- :sync_error nil :draft false)
- :changed? (app-content-changed? existing fields)}))
- (catch Throwable e
- (log/warnf "[data-app] failed to read bundle for app %s: %s" slug (ex-message e))
- {:fields (assoc metadata :sync_error (ex-message e))
- :changed? (or (nil? existing)
- (not= (:sync_error existing) (ex-message e))
- (app-metadata-changed? existing metadata))}))))
-
-(defn- record-app-failure!
- "Record a failure after the app transaction rolls back. Keep existing content and resource links intact."
- [existing slug fields message]
- (try
- (t2/with-transaction [_conn]
- (if existing
- (mark-config-error! existing slug message)
- (when fields
- (upsert-by-name! slug (assoc (select-keys fields synced-metadata-keys)
- :sync_error message))
- true)))
- (catch Throwable e
- (log/warnf "[data-app] could not record sync error for app %s: %s" slug (ex-message e))
- false)))
-
-(defn- sync-app!
- "Commit one app and its permission resources together. Return whether its recorded state changed."
- [existing {:keys [slug config-error] :as cfg}]
- (let [{:keys [fields changed?]} (when-not config-error (prepare-app existing cfg))]
- (try
- (perms/with-global-permissions-lock
- (t2/with-transaction [_conn]
- (data-apps.db/publish-data-app-drafts! [slug])
- (if config-error
- (mark-config-error! existing slug config-error)
- (do
- (upsert-by-name! slug fields)
- (data-app.resources/ensure-resources! (data-apps.db/non-blob-data-app-by-slug slug))
- changed?))))
- (catch Throwable e
- (log/warnf "[data-app] failed to sync app %s: %s" slug (ex-message e))
- (record-app-failure! existing slug fields (ex-message e))))))
-
-(defn- prune-apps!
- "Delete all absent apps. Preserve the sync summary if pruning rolls back."
- [present-slugs]
- (try
- {:removed (t2/with-transaction [_conn]
- (if (seq present-slugs)
- (data-apps.db/delete-data-apps-not-named! present-slugs)
- (data-apps.db/delete-all-data-apps!)))}
- (catch Throwable e
- (log/warnf "[data-app] pruning failed: %s" (ex-message e))
- {:removed 0 :pruning-error (ex-message e)})))
-
-(defn import-from-snapshot!
- "Materialize data apps from a synced repo `snapshot`:
-
- {:read-file <(fn [path] -> <file-text-string> | nil)>
- :list-dir <(fn [path] -> [<child-path-string> ...])>
- :sha <commit-sha-string>}
-
- Discovers every `data_apps/<dir>/data_app.yaml`, upserts a row per app, and prunes
- rows whose directory is gone from the snapshot. Apps are synced independently, so
- one failing app cannot roll back the others. Returns
- `{:synced <n>, :changed <n>, :removed <n>, :sha <sha>, :config-errors [<msg> ...]}`,
- where `:changed` counts apps actually created/updated (a `last_synced_sha` bump on
- unchanged content does not count) and `:removed` counts apps dropped for no longer
- being in the repo. If pruning fails, `:removed` is zero and `:pruning-error` contains
- the error message. Counts for committed app updates are preserved.
-
- Two apps can't collide on a slug here: a slug *is* an app's directory name (see
- the config namespace), a repo can't hold two `data_apps/<slug>` directories, and
- discovery takes one config per directory (see [[discover-app-configs]])."
- [{:keys [read-file list-dir sha]}]
- ;; realize discovery once (it calls read-file/parse per config); `results` is
- ;; then walked several times below
- (let [results (vec (discover-app-configs list-dir read-file))
- good (remove :config-error results)
- errors (vec (keep :config-error results))
- ;; every app directory present in the repo (parsed or not) — the set of
- ;; slugs that should survive this sync
- present-slugs (into #{} (map :slug) results)
- ;; pre-sync rows, so we can tell a real change from a sha/timestamp bump
- existing (into {} (map (juxt :name identity))
- (data-apps.db/data-apps-sync-info))
- changed (count (filter (fn [{:keys [slug] :as cfg}]
- (sync-app! (get existing slug) (assoc cfg :sha sha :read-file read-file)))
- results))
- {:keys [removed] :as pruning-result} (prune-apps! present-slugs)]
- (log/infof "[data-app] synced sha=%s apps=%d changed=%d removed=%d errors=%d"
- sha (count good) changed removed (count errors))
- (merge {:synced (count good), :changed changed, :sha sha, :config-errors errors}
- pruning-result)))
-
-(defn sync-from-snapshot!
- "Entry point for the remote-sync import pipeline. Materializes data apps from
- the just-imported `snapshot` (see [[import-from-snapshot!]] for its shape).
- Never throws, so it can't break the surrounding remote-sync import: a thrown
- failure is logged and swallowed, and a malformed `data_app.yaml` is logged and
- its app simply doesn't appear. Returns the [[import-from-snapshot!]] result, or
- nil if the sync threw.
-
- A no-op returning nil when the `:data-apps` feature is absent: without
- it an instance behaves exactly as if data apps did not exist — no rows,
- collections, or permission groups are materialized, and nothing is pruned."
- [snapshot]
- (when (premium-features/enable-data-apps?)
- (try
- (let [{:keys [config-errors] :as result} (import-from-snapshot! snapshot)]
- (when (seq config-errors)
- (log/warnf "[data-app] %d data_app.yaml config(s) skipped: %s"
- (count config-errors) (str/join "; " config-errors)))
- result)
- (catch Throwable e
- (log/warnf "[data-app] sync from remote-sync snapshot failed: %s" (ex-message e))
- nil))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
index 26f72f41a61..855fc93a8a1 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
@@ -13,7 +13,7 @@
(def ^:private ConditionKey
"The column keys used in the `:conditions` / `:cascade-filter` / `:removal-conditions` of a remote-sync model
spec (see `metabase-enterprise.remote-sync.spec`)."
- [:enum :exploration_id :built_in_type :active :entity_id :collection_id :archived :archived_at])
+ [:enum :exploration_id :built_in_type :active :entity_id :collection_id :archived :archived_at :draft])
(def ^:private Conditions
"A map of column to value (possibly nil) or Toucan 2 operator-vector value, or nil for none."
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 282b95eeb3e..172cdbd4a86 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -3,7 +3,6 @@
[clojure.string :as str]
[diehard.core :as dh]
[java-time.api :as t]
- [metabase-enterprise.data-apps.sync :as data-apps.sync]
[metabase-enterprise.remote-sync.db :as remote-sync.db]
[metabase-enterprise.remote-sync.guards :as guards]
[metabase-enterprise.remote-sync.merge :as remote-sync.merge]
@@ -260,7 +259,7 @@
{:model_type model-type
:model_id (:id instance)
:path (or repo-path (:path fspec))
- :content_hash (source/content-hash (:content fspec))})
+ :content_hash (source/file-spec-hash fspec)})
(catch Exception e
(log/warnf "Skipping %s %s: failed to serialize for content hash: %s" model-type (:id instance) (ex-message e))
nil)))]
@@ -308,21 +307,6 @@
(and (some? pre-task-branch)
(not= pre-task-branch (settings/remote-sync-branch))))
-(defn- materialize-data-apps!
- "Materialize data apps from the snapshot a content import is landing. Data apps live under `data_apps/` in
- the repo (outside the serdes paths), so they ride this import rather than having their own sync. Runs inside
- the import's commit transaction, before the version is written: a task's `version` is the sync base (see
- [[remote-sync.db/last-synced-task]]) and gates every later pull, so a version that landed without its data
- apps would never be re-materialized. Adapts the snapshot to plain reader fns;
- `data-apps.sync/sync-from-snapshot!` never throws."
- [^SourceSnapshot snapshot]
- ;; `read-file` returns file text (a string) or nil; data-apps.sync converts to
- ;; bytes on its side, keeping all Java interop out of this namespace.
- (data-apps.sync/sync-from-snapshot!
- {:read-file (fn [path] (source.p/read-file snapshot path))
- :list-dir (fn [path] (source.p/list-dir snapshot path))
- :sha (source.p/version snapshot)}))
-
(defn- import-progress-reporter
"A throttled progress reporter (see `make-progress-reporter`) for the import task `task-id`."
[task-id]
@@ -397,41 +381,31 @@
"Models whose change forces a full import on the incremental fast-path: Collection (a rename moves every
descendant's file, a delete cascades to its contents) and the feature models (their presence drives the
remote-sync-transforms / library settings, which need whole-snapshot knowledge to toggle correctly)."
- #{"Collection" "Transform" "TransformTag" "PythonLibrary" "NativeQuerySnippet"})
+ #{"Collection" "DataApp" "Transform" "TransformTag" "PythonLibrary" "NativeQuerySnippet"})
-(defn- legal-yaml-path?
- "True for a managed-directory `.yaml` entity file — the only changed paths the importer acts on."
+(defn- managed-path?
+ "True for a file under a managed directory."
[^String path]
- (and (str/ends-with? path ".yaml")
- (not (str/starts-with? path "."))
+ (and (not (str/starts-with? path "."))
(when-let [i (str/index-of path "/")]
(contains? serialization/legal-top-level-paths (subs path 0 i)))))
+(defn- legal-yaml-path?
+ "True for a managed-directory `.yaml` entity file — the only changed paths the importer acts on."
+ [^String path]
+ (serialization/entity-file-path? path))
+
(defn- pulled-change-count
"Total number of entities applied by a pull, across entity-id- and path-identified models in `imported-data`."
[imported-data]
(transduce (map count) + 0 (concat (vals (:by-entity-id imported-data))
(vals (:by-path imported-data)))))
-(defn- fold-data-app-changes
- "Fold the count of data apps a pull changed — upserted *or* removed — into its
- `:outcome`. Data apps live under `data_apps/` — outside serdes — and are
- materialized separately (see [[materialize-data-apps!]]), so they're invisible
- to [[pulled-change-count]]. Without this, a pull whose only changes are data
- apps (including one that only deletes an app) reports `pull-skipped` /
- `count 0`. Caller guarantees `da-changed` is positive."
- [outcome da-changed]
- (case (:kind outcome)
- "pull-skipped" {:kind "pulled" :count da-changed :branch (settings/remote-sync-branch)}
- "pulled" (update outcome :count (fnil + 0) da-changed)
- "merged" (update outcome :pulled (fnil + 0) da-changed)
- outcome))
-
(defn- incremental-import-plan
"What an incremental load would touch, or [[incremental-not-possible]] if the change must fall back to a full
import. On success returns `{:ingestable <ingestable-or-nil> :deleted-rsos <RemoteSyncObject rows>}`. Falls back
- when there is no diff, a deleted file can't be mapped back to a tracked entity, or a
- structural/feature/non-entity-id model changed."
+ when there is no diff, a resource file changed or a non-YAML file was deleted, a deleted file can't be mapped back
+ to a tracked entity, or a structural/feature/non-entity-id model changed."
[snapshot last-version]
(let [changed (when last-version (source.p/changed-files snapshot last-version))
add-mod (into #{} (filter legal-yaml-path?) (into (:added changed) (:modified changed)))
@@ -449,6 +423,10 @@
(nil? changed) ;; first import, or diffing not possible (force-push/rebase, non-diffable source)
:remote-sync/incremental-not-possible
+ (or (seq (source/owned-paths snapshot (remove legal-yaml-path? (concat (:added changed) (:modified changed)))))
+ (some #(and (managed-path? %) (not (legal-yaml-path? %))) (:deleted changed)))
+ :remote-sync/incremental-not-possible
+
(some nil? deleted-rsos) ;; some deleted file not present in appdb
:remote-sync/incremental-not-possible
@@ -602,11 +580,7 @@
first-import? (nil? last-imported-version)
;; force-deletion? defaults to force? when a caller doesn't pass it.
force-deletion? (if (nil? force-deletion?) force? force-deletion?)
- da-result (volatile! nil)
- ;; Data apps before the version: the version write locks the task row until commit, which would
- ;; block the heartbeat for the whole materialization.
finalize! (fn []
- (vreset! da-result (materialize-data-apps! snapshot))
(remote-sync.task/set-version! task-id snapshot-version))
report (import-progress-reporter task-id)
path-filters (mapv #(re-pattern (str % "/.*")) serialization/legal-top-level-paths)
@@ -711,18 +685,7 @@
:outcome {:kind "pulled"
:count (pulled-change-count imported-data)
:branch (settings/remote-sync-branch)}}))]
- ;; Data apps rode the pull inside `finalize!`, materialized from the real source snapshot (the repo
- ;; file tree under `data_apps/`), not the synthetic merged snapshot `load-snapshot!` sees. They're
- ;; counted outside serdes, so fold how many they upserted or removed into the outcome — otherwise a
- ;; data-app-only pull would report `pull-skipped` / `count 0`. Paths that skipped `finalize!` left
- ;; no result: the version did not move, so neither did the apps.
- (if (= :success (:status result))
- ;; Removals count too: a pull whose only change is deleting an app directory upserts nothing
- ;; (`:changed` 0) but still changed what this instance serves, so it must report as a pull, not skipped.
- (let [da-changed (+ (:changed @da-result 0) (:removed @da-result 0))]
- (cond-> result
- (pos? da-changed) (update :outcome fold-data-app-changes da-changed)))
- result))
+ result)
(catch Exception e
;; A cancellation isn't a failure: log it and return nil. Otherwise log the error, count it, and
;; return a user-friendly :error result.
@@ -789,7 +752,7 @@
:message "Export blocked: the same content was changed both locally and on the remote branch."})
(let [[_ version] (commit-staged! snapshot message
(fn [commit]
- (source.p/replace-all! commit) ; merged set replaces the managed dirs wholesale
+ (source/replace-managed-files! commit snapshot) ; merged set replaces the managed files wholesale
(run! #(source.p/stage-upsert! commit %) merged)))
;; An empty merge means the merged set already matched the remote tip: nothing was pushed, so
;; reconcile (and advance) against the tip rather than a non-existent merge commit.
@@ -955,6 +918,9 @@
(:identity (spec/spec-for-model-type model_type)))
:remote-sync/incremental-not-possible
+ (:resources? (spec/spec-for-model-type model_type))
+ :remote-sync/incremental-not-possible
+
(not (#{"create" "update" "removed" "delete"} status))
:remote-sync/incremental-not-possible
@@ -1075,11 +1041,11 @@
(not (settings/library-is-remote-synced?)) (into ["snippets" "glossary"])))
(defn- stage-write [commit opts [row entity]]
- (let [path (or (:file_path row) (source/entity->path opts entity))
- content (source/entity->content entity)]
- (source.p/stage-upsert! commit {:path path :content content})
+ (let [path (or (:file_path row) (source/entity->path opts entity))
+ fspec (source/entity->file-spec-at path entity)]
+ (run! #(source.p/stage-upsert! commit %) (source/file-specs fspec))
(when (:id row)
- [{:id (:id row) :file_path path :content_hash (source/content-hash content)}])))
+ [{:id (:id row) :file_path path :content_hash (source/file-spec-hash fspec)}])))
(defn- chunk-stage-writes
[commit opts chunk]
@@ -1157,7 +1123,7 @@
(let [opts (source/storage-context)
[synced version] (commit-staged! snapshot message
(fn [commit]
- (source.p/replace-all! commit) ; replace the managed dirs wholesale
+ (source/replace-managed-files! commit snapshot) ; replace the managed files wholesale
(let [synced (stage-writes commit opts export-rows
(fn [staged]
(report (+ export-progress-plan-done
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj
index b2c877b0132..a281509bd6d 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj
@@ -17,6 +17,7 @@
and produces a merged set of file specs plus a list of genuine conflicts (the same entity changed
differently on both sides)."
(:require
+ [clojure.string :as str]
[metabase.models.serialization :as serdes]
[metabase.util.log :as log]
[metabase.util.yaml :as yaml]))
@@ -41,10 +42,11 @@
nil)))
(defn- file-key
- "Identity key for a `{:path :content}` file spec: the serdes identity when available, otherwise a
- path-based fallback so non-serdes files still merge sanely."
- [{:keys [path content]}]
- (or (entity-identity content)
+ "Identity key for a `{:path :content}` file spec: the serdes identity of a YAML file when available, otherwise a
+ path-based fallback so non-serdes files, such as resource files, still merge sanely."
+ [{:keys [^String path content]}]
+ (or (when (str/ends-with? path ".yaml")
+ (entity-identity content))
[::by-path path]))
(defn- index-by-key
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
index 6fbd471c03d..33de53b88d4 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
@@ -84,28 +84,51 @@
(defn entity->content
"The serialized YAML string for an extracted `entity`."
[entity]
- (yaml/generate-string (serialization/serialization-deep-sort entity)
+ (yaml/generate-string (serialization/serialization-deep-sort (serialization/without-resources entity))
{:dumper-options {:flow-style :block :split-lines false}}))
+(defn- resource-specs
+ "The `{:path :content}` file specs of `entity`'s resource files, next to its YAML file at `path`."
+ [path entity]
+ (let [dir (subs path 0 (inc (or (str/last-index-of path "/") -1)))]
+ (for [[resource content] (sort-by key (:serdes/resources entity))]
+ {:path (str dir resource) :content content})))
+
+(defn entity->file-spec-at
+ "Serializes a single extracted entity at the YAML `path` into a `{:path :content :resources}` file spec;
+ `:resources` holds the `{:path :content}` specs of its resource files."
+ [path entity]
+ {:path path
+ :content (entity->content entity)
+ :resources (vec (resource-specs path entity))})
+
(defn entity->file-spec
- "Serializes a single extracted entity into a `{:path :content}` file spec, using storage context
- `opts` (from [[storage-context]])."
+ "[[entity->file-spec-at]] the path storage context `opts` (from [[storage-context]]) gives `entity`."
[opts entity]
- {:path (entity->path opts entity)
- :content (entity->content entity)})
+ (entity->file-spec-at (entity->path opts entity) entity))
+
+(defn file-specs
+ "The `{:path :content}` specs of every file a `file-spec` from [[entity->file-spec]] writes."
+ [file-spec]
+ (cons (select-keys file-spec [:path :content]) (:resources file-spec)))
(defn content-hash
"SHA-256 (hex) of a serialized YAML `content` string."
[^String content]
(codecs/bytes->hex (buddy-hash/sha256 content)))
+(defn file-spec-hash
+ "SHA-256 (hex) of the content of a `file-spec` from [[entity->file-spec]], including its resource files."
+ [{:keys [content resources]}]
+ (content-hash (apply str content (mapcat (juxt :path :content) resources))))
+
(defn row->file-info
"The repo `:path` and the SHA-256 (hex) `:content-hash` of the serialized YAML for the entity named by `row`
({:model_type :model_id}), or nil if it can't be extracted."
[row]
(when-let [entity (first (spec/extract-entities-for-rows [row]))]
- (let [{:keys [path content]} (entity->file-spec (storage-context) entity)]
- {:path path, :content-hash (content-hash content)})))
+ (let [fspec (entity->file-spec (storage-context) entity)]
+ {:path (:path fspec), :content-hash (file-spec-hash fspec)})))
(defn row->content-hash
"SHA-256 (hex) of the serialized YAML for the entity named by `row` ({:model_type :model_id}), or nil if it
@@ -132,26 +155,70 @@
(remote-sync.task/update-progress! task-id (-> n (/ total) (min 1) (* 0.65) (+ 0.3))))
(constantly nil))]
(into []
- (map-indexed (fn [idx entity]
- (when (instance? Exception entity)
- (throw entity))
- (let [spec (entity->file-spec opts entity)]
- (report (inc idx))
- spec)))
+ (comp (map-indexed (fn [idx entity]
+ (when (instance? Exception entity)
+ (throw entity))
+ (let [spec (entity->file-spec opts entity)]
+ (report (inc idx))
+ (file-specs spec))))
+ cat)
stream)))
+(defn- managed-path? [^String path]
+ (when-let [idx (str/index-of path "/")]
+ (contains? serialization/legal-top-level-paths (subs path 0 idx))))
+
+(defn- shared-path? [^String path]
+ (when-let [idx (str/index-of path "/")]
+ (contains? serialization/shared-top-level-paths (subs path 0 idx))))
+
+(defn- parent-dir [^String path]
+ (when-let [idx (str/last-index-of path "/")]
+ (subs path 0 idx)))
+
+(defn- declared-resources
+ "The paths of the resource files that the entity YAML files directly in directory `dir` of `snapshot` declare."
+ [snapshot dir]
+ (into #{}
+ (comp (filter serialization/entity-file-path?)
+ (mapcat (fn [path]
+ (try
+ (let [entity (yaml/parse-string (source.p/read-file snapshot path)
+ {:key-fn serialization/parse-key})]
+ (map #(str dir "/" %) (serdes/resource-paths entity)))
+ (catch Exception _
+ nil)))))
+ (source.p/list-dir snapshot dir)))
+
+(defn owned-paths
+ "The managed paths among `paths` that serialization owns in `snapshot`: entity YAML files, and the resource files
+ that a YAML file in one of their ancestor directories declares."
+ [snapshot paths]
+ (let [resources (memoize #(declared-resources snapshot %))]
+ (filterv (fn [path]
+ (and (managed-path? path)
+ (or (serialization/entity-file-path? path)
+ (some #(contains? (resources %) path)
+ (take-while some? (iterate parent-dir (parent-dir path)))))))
+ paths)))
+
+(defn replace-managed-files!
+ "Stage the removal of every file serialization owns in `snapshot` into `commit`, so the files staged next replace
+ them wholesale; files in [[serialization/shared-top-level-paths]] that serialization does not own are kept."
+ [commit snapshot]
+ (source.p/replace-all! commit)
+ (run! #(source.p/stage-delete! commit %)
+ (owned-paths snapshot (filter shared-path? (source.p/list-files snapshot)))))
+
(defn- snapshot->specs
- "Reads a snapshot's managed-directory files into a sequence of `{:path :content}` specs, matching the
+ "Reads the files serialization owns in a snapshot into a sequence of `{:path :content}` specs, matching the
shape produced by [[serialize-specs]]. Used to read the merge base and remote-tip trees for merging."
[snapshot]
(into []
(keep (fn [path]
- (when (contains? serialization/legal-top-level-paths
- (when-let [idx (str/index-of path "/")]
- (subs path 0 idx)))
- (when-let [content (source.p/read-file snapshot path)]
- {:path path :content content}))))
- (source.p/list-files snapshot)))
+ (when-let [content (source.p/read-file snapshot path)]
+ {:path path :content content})))
+ (owned-paths snapshot (source.p/list-files snapshot))))
(defn compute-merge
"Runs the entity-identity 3-way merge of local state against the remote tip, without writing. Returns
@@ -218,6 +285,6 @@
(setting/get :remote-sync-url)
(or branch (setting/get :remote-sync-branch))
(setting/get :remote-sync-token)
- serialization/legal-top-level-paths))
+ serialization/replaced-top-level-paths))
([]
(source-from-settings (setting/get :remote-sync-branch))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
index c643071bf5c..0111ef8fe8d 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
@@ -40,8 +40,7 @@
[snapshot]
(let [errors (atom [])]
{:entities (into {} (for [path (source.p/list-files snapshot)
- :when (and (not (str/starts-with? path "."))
- (str/ends-with? path ".yaml"))
+ :when (serialization/entity-file-path? path)
:let [content (try
(source.p/read-file snapshot path)
(catch Exception e
@@ -147,9 +146,11 @@
(ingest-one [_ serdes-path]
(populate-cache! cache errors-atom #(ingest-all snapshot))
- (when-let [target (get @cache (serialization/strip-labels serdes-path))]
+ (when-let [{:keys [content ^String path]} (get @cache (serialization/strip-labels serdes-path))]
(try
- (ingest-content (:content target))
+ (let [dir (subs path 0 (inc (or (str/last-index-of path "/") -1)))]
+ (serialization/read-resources (ingest-content content)
+ #(source.p/read-file snapshot (str dir %))))
(catch Exception e
(throw (ex-info "Unable to ingest file" {:abs-path serdes-path} e))))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 26814669d1d..c4735e9f391 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -50,8 +50,9 @@
:prefix - Event keyword prefix (e.g., :event/card)
:types - Vector of event types to handle [:create :update :delete]
- :eligibility - Eligibility configuration:
- :type - :collection, :published-table, :parent-table, :parent, :setting, or
- :library-synced (:parent follows :parent-fk to a :parent-model instance)
+ :type - :collection, :published-table, :parent-table, :parent, :setting,
+ :library-synced, or :always (:parent follows :parent-fk to a :parent-model
+ instance)
:collection - For :collection type: :remote-synced, :transforms-namespace, :snippets-namespace, or :any
:setting - For :setting type: setting keyword to check
(Note: :library-synced type uses the library-is-remote-synced? setting to determine eligibility)
@@ -76,6 +77,9 @@
then scoped to that table's rows in the synced collections (Action)
:all-on-setting-disable - Optional setting keyword; when this setting's sentinel
RSO exists with 'delete' status, remove ALL entities of this type
+ - :resources? - Optional. True when the model's YAML files carry resource files (see
+ `metabase.models.serialization/resource-paths`), whose changes only a full import or export
+ handles.
- :export-scope - Export scope for query-export-roots:
:root-collections - Query root-level remote-synced + namespace collections (Collection)
:root-only - Query root instances with collection_id = nil (Transform)
@@ -364,7 +368,23 @@
:removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion
:all-on-setting-disable :remote-sync-transforms}
:export-scope :all ; query for all instances
- :enabled? :remote-sync-transforms}})
+ :enabled? :remote-sync-transforms}
+
+ :model/DataApp
+ {:model-type "DataApp"
+ :model-key :model/DataApp
+ :identity :entity-id
+ :events {:prefix :event/data-app
+ :types [:create :update :delete]}
+ :eligibility {:type :always}
+ :archived-key nil
+ :tracking {:select-fields [:name]
+ :field-mappings {:model_name :name}}
+ :conditions {:draft false}
+ :removal {:statuses #{"removed" "delete"}}
+ :resources? true
+ :export-scope :all
+ :enabled? true}})
;;; ------------------------------------------------- Helper Functions -------------------------------------------------
@@ -742,29 +762,30 @@
({:type :category :model :count :names :message}), one per affected model type."
[{:keys [by-entity-id]}]
(let [synced-collection-ids (all-syncable-collection-ids)]
- (cond-> []
- (seq synced-collection-ids)
- (into (for [[model-key spec] (specs-for-deletion)
- :when (and (not (get-in spec [:removal :all-on-setting-disable]))
- (not= :model/Collection model-key))
- :let [model-type (:model-type spec)
- imported-ids (get by-entity-id model-type #{})
- ;; Same base restriction remove-unsynced! deletes by, plus an anti-join keeping only the
- ;; unsynced rows the import would delete. Done in SQL so we never materialize a whole
- ;; collection's worth of rows just to count/sample them.
- opts (removal-opts spec synced-collection-ids imported-ids)
- n (remote-sync.db/unsynced-instance-count model-key model-type opts)
- name-col (get-in spec [:tracking :field-mappings :model_name])]
- :when (pos? n)]
- {:type (keyword (str (u/lower-case-en model-type) "-deletion-conflict"))
- :category model-type
- :model model-type
- :count n
- ;; A bounded sample of names for the UI; :count above is the true total.
- :names (remote-sync.db/unsynced-instance-names model-key model-type name-col opts
- max-conflict-names)
- :message (format "Import would delete %d unsynced local %s %s"
- n model-type (if (= 1 n) "entity" "entities"))})))))
+ (into []
+ (for [[model-key spec] (specs-for-deletion)
+ :when (and (not (get-in spec [:removal :all-on-setting-disable]))
+ (not= :model/Collection model-key)
+ (or (seq synced-collection-ids)
+ (nil? (get-in spec [:removal :scope-key]))))
+ :let [model-type (:model-type spec)
+ imported-ids (get by-entity-id model-type #{})
+ ;; Same base restriction remove-unsynced! deletes by, plus an anti-join keeping only the
+ ;; unsynced rows the import would delete. Done in SQL so we never materialize a whole
+ ;; collection's worth of rows just to count/sample them.
+ opts (removal-opts spec synced-collection-ids imported-ids)
+ n (remote-sync.db/unsynced-instance-count model-key model-type opts)
+ name-col (get-in spec [:tracking :field-mappings :model_name])]
+ :when (pos? n)]
+ {:type (keyword (str (u/lower-case-en model-type) "-deletion-conflict"))
+ :category model-type
+ :model model-type
+ :count n
+ ;; A bounded sample of names for the UI; :count above is the true total.
+ :names (remote-sync.db/unsynced-instance-names model-key model-type name-col opts
+ max-conflict-names)
+ :message (format "Import would delete %d unsynced local %s %s"
+ n model-type (if (= 1 n) "entity" "entities"))}))))
(defn- object-matches-conditions?
"True if `object` satisfies every column-value pair in `conditions` (or if no conditions are set).
@@ -844,6 +865,10 @@
[_spec _object]
(rs-settings/library-is-remote-synced?))
+(defmethod check-eligibility-by-type :always
+ [_ _]
+ true)
+
(defmethod check-eligibility-by-type :default
[_ _]
false)
@@ -1238,6 +1263,15 @@
(remote-sync.db/ids-where model-key (when archived-key {archived-key false})))
nil)))
+(defmethod query-export-roots :always
+ [{:keys [export-scope model-key model-type] :as spec}]
+ (case export-scope
+ :all
+ (into #{}
+ (map (fn [id] [model-type id]))
+ (remote-sync.db/ids-where model-key (export-conditions spec)))
+ nil))
+
(defmethod query-export-roots :default [_] nil)
(def git-sync-extract-opts
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/core.clj b/enterprise/backend/src/metabase_enterprise/serialization/core.clj
index a1a88a32d9e..ec11ccfe9f9 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/core.clj
@@ -25,13 +25,17 @@
make-targets-of-type
extract]
[metabase-enterprise.serialization.v2.ingest
+ entity-file-path?
legal-top-level-paths
+ replaced-top-level-paths
+ shared-top-level-paths
strip-labels
Ingestable
ingest-yaml
ingest-list
ingest-one
ingest-errors
+ read-resources
read-timestamps
parse-key]
[metabase-enterprise.serialization.v2.load
@@ -42,4 +46,6 @@
file-writer]
[metabase-enterprise.serialization.v2.storage.util
resolve-storage-path
- slugify-name])
+ resource-files
+ slugify-name
+ without-resources])
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
index cb35344cf1c..d745db7af81 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
@@ -40,7 +40,10 @@
(conj "EmbeddingTheme")
(not (:no-custom-viz-plugins opts))
- (conj "CustomVizPlugin")))
+ (conj "CustomVizPlugin")
+
+ (not (:no-data-apps opts))
+ (conj "DataApp")))
;; OsiAiContext is intentionally NOT in the default export set. It's a top-level model that *depends on* its
;; entity, extracted unfiltered, so any export — even an untargeted "full" one, which still omits personal and
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
index ce05c58c158..b2fb591cdb5 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
@@ -80,10 +80,54 @@
(yaml/from-file {:key-fn parse-key})
read-timestamps))
+(defn- check-resource-path!
+ "Throws unless `path` is a relative path that stays inside its entity's directory."
+ [path]
+ (when-not (and (string? path)
+ (not (str/starts-with? path "/"))
+ (not (str/includes? path "\\"))
+ (not-any? #{"" "." ".."} (str/split path #"/" -1)))
+ (throw (ex-info (format "Invalid resource file path: %s" path) {:path path}))))
+
+(defn read-resources
+ "The ingested `entity` with each of its [[serdes/resource-paths]] read by `read-fn` into `:serdes/resources`."
+ [entity read-fn]
+ (if-let [paths (seq (serdes/resource-paths entity))]
+ (assoc entity :serdes/resources (into {}
+ (map (fn [path]
+ (check-resource-path! path)
+ [path (read-fn path)]))
+ paths))
+ entity))
+
(def legal-top-level-paths
"Known top-level paths for directory with serialization output.
We support both \"python-libraries\" and \"python_libraries\" for backwards compatibility. The modern name is \"python_libraries\"."
- #{"actions" "channels" "collections" "custom_viz_plugins" "databases" "embedding_themes" "glossary" "metabots" "python_libraries" "python-libraries" "osi_ai_context" "snippets" "transforms"})
+ #{"actions" "channels" "collections" "custom_viz_plugins" "data_apps" "databases" "embedding_themes" "glossary" "metabots" "python_libraries" "python-libraries" "osi_ai_context" "snippets" "transforms"})
+
+(def shared-top-level-paths
+ "The [[legal-top-level-paths]] whose directories also hold files serialization does not own, such as a data app's
+ source code, to the name of the entity files in them."
+ {"data_apps" "data_app.yaml"})
+
+(def replaced-top-level-paths
+ "The [[legal-top-level-paths]] a full export may replace wholesale."
+ (apply disj legal-top-level-paths (keys shared-top-level-paths)))
+
+(defn entity-file-path?
+ "Whether the `/`-separated relative `path` names an entity YAML file: a `.yaml` file that isn't a dotfile under one
+ of the [[legal-top-level-paths]], with the entity file name of its directory when that is one of the
+ [[shared-top-level-paths]]."
+ [^String path]
+ (let [top (first (str/split path #"/" 2))
+ file-name (subs path (inc (or (str/last-index-of path "/") -1)))]
+ (boolean (and (str/includes? path "/")
+ (str/ends-with? file-name ".yaml")
+ (not (str/starts-with? file-name "."))
+ (contains? legal-top-level-paths top)
+ (if-let [entity-file-name (get shared-top-level-paths top)]
+ (= entity-file-name file-name)
+ true)))))
(defn- path-interner
"Returns a function that interns `:serdes/meta` path vectors.
@@ -104,14 +148,11 @@
(mapv intern-seg hierarchy))))
(defn- ingestible-file?
- "Whether `file` is a regular `.yaml` file under one of the [[legal-top-level-paths]].
- Dotfiles are excluded (editor temp files, see #41567)."
+ "Whether `file` is a regular file that [[entity-file-path?]] accepts. Dotfiles are excluded (editor temp files, see
+ #41567)."
[^File root-dir ^File file]
- (boolean (and (.isFile file)
- (not (str/starts-with? (.getName file) "."))
- (str/ends-with? (.getName file) ".yaml")
- (let [rel (.relativize (.toPath root-dir) (.toPath file))]
- (-> rel (.subpath 0 1) (.toString) legal-top-level-paths)))))
+ (and (.isFile file)
+ (entity-file-path? (str/replace (str (.relativize (.toPath root-dir) (.toPath file))) "\\" "/"))))
(defn- file-hierarchy!
"Parses `file` and returns its `:serdes/meta` abstract path, or nil on parse failure.
@@ -164,7 +205,11 @@
{:serdes/meta serdes-meta :key kw-id :value (get settings kw-id)})
(when-let [file (get @cache (strip-labels serdes-meta))]
(try
- (ingest-file file)
+ (read-resources (ingest-file file)
+ (fn [path]
+ (let [resource (io/file (.getParentFile ^File file) ^String path)]
+ (when (.isFile resource)
+ (slurp resource)))))
(catch Exception e
(throw (ex-info "Unable to ingest file" {:file (.getName ^File file)
:abs-path serdes-meta} e))))))))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index 44cafccea21..0c12065c267 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -35,6 +35,7 @@
(concat data-model
content
["CustomVizPlugin"
+ "DataApp"
"EmbeddingTheme"
"FieldValues"
"Metabot"
@@ -84,7 +85,6 @@
"ConnectionImpersonation"
"ContentTranslation"
"DashboardBookmark"
- "DataApp"
"DataComplexityScore"
"DataPermissions"
"DatabaseRouter"
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj
index 92bdff665de..06fb6485ef8 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj
@@ -13,11 +13,8 @@
(set! *warn-on-reflection* true)
-(defn- file ^File [ctx root-dir entity]
- (let [resolved (storage.util/resolve-storage-path ctx entity)
- dirnames (drop-last resolved)
- basename (str (last resolved) ".yaml")]
- (apply io/file root-dir (concat dirnames [basename]))))
+(defn- yaml-file ^File [root-dir resolved]
+ (apply io/file root-dir (concat (drop-last resolved) [(str (last resolved) ".yaml")])))
(defn file-writer
"Create a filesystem storage backend rooted at `root-dir`."
@@ -25,11 +22,16 @@
(let [ctx (serdes/storage-base-context)]
(reify protocols/ExportWriter
(store-entity! [_ entity]
- (let [f (file ctx root-dir entity)]
+ (let [resolved (storage.util/resolve-storage-path ctx entity)
+ f (yaml-file root-dir resolved)]
(log/info "Storing" {:path (serdes/log-path-str (:serdes/meta entity))
:file (str (.relativize (Path/of (str root-dir) (make-array String 0))
(.toPath f)))})
- (spit-yaml! f entity)
+ (spit-yaml! f (storage.util/without-resources entity))
+ (doseq [[segments content] (storage.util/resource-files resolved entity)
+ :let [resource (apply io/file root-dir segments)]]
+ (io/make-parents resource)
+ (spit resource content))
(:serdes/meta entity)))
(store-settings! [_ settings]
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj
index 368a925bcc7..277218f5331 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj
@@ -41,9 +41,11 @@
(store-entity! [_ entity]
(let [resolved (storage.util/resolve-storage-path ctx entity)
path (entry-path dirname resolved)
- content (.getBytes ^String (yaml-content entity) "UTF-8")]
+ content (.getBytes ^String (yaml-content (storage.util/without-resources entity)) "UTF-8")]
(log/trace "Storing" {:path (serdes/log-path-str (:serdes/meta entity))})
(put-entry! tar path content)
+ (doseq [[segments ^String resource] (storage.util/resource-files resolved entity)]
+ (put-entry! tar (str dirname "/" (str/join "/" segments)) (.getBytes resource "UTF-8")))
(:serdes/meta entity)))
(store-settings! [_ settings]
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj
index 0774eff3303..3bd6adbc7a6 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj
@@ -25,15 +25,17 @@
(u.str/limit-chars max-label-length))))
(defn- resolve-path
- "Given a storage path (vector of `{:label ... :key ...}` maps), resolves to a vector of strings
+ "Given a storage path (vector of `{:label ... :key ... :style ...}` maps), resolves to a vector of strings
with deduplication per folder."
[unique-name-fns path]
(loop [remaining path
resolved []]
(if (empty? remaining)
resolved
- (let [{:keys [label key]} (first remaining)
- slug (slugify-name label)
+ (let [{:keys [label key style] :or {style :name}} (first remaining)
+ slug (case style
+ :name (slugify-name label)
+ :slug label)
gen (or (get @unique-name-fns resolved)
(let [g (lib/non-truncating-unique-name-generator)]
(swap! unique-name-fns assoc resolved g)
@@ -47,3 +49,14 @@
The last element is the filename (without extension)."
[ctx entity]
(resolve-path (:unique-name-fns ctx) (serdes/storage-path entity ctx)))
+
+(defn without-resources
+ "The `entity` as written to its YAML file, without its `:serdes/resources`."
+ [entity]
+ (dissoc entity :serdes/resources))
+
+(defn resource-files
+ "Each of `entity`'s `:serdes/resources` as `[path-segments content]`, next to its YAML file at `resolved`."
+ [resolved entity]
+ (for [[path content] (sort-by key (:serdes/resources entity))]
+ [(into (vec (drop-last resolved)) (str/split path #"/")) content]))
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj
index e7621111f0a..6aa15047926 100644
--- a/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj
@@ -2,12 +2,11 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
+ [metabase-enterprise.data-apps.apps :as data-apps.apps]
[metabase-enterprise.data-apps.config :as data-app.config]
[metabase-enterprise.data-apps.query-definition :as query-definition]
[metabase-enterprise.data-apps.resources :as data-app.resources]
- [metabase-enterprise.data-apps.sync :as data-app.sync]
[metabase-enterprise.data-apps.user-access :as data-app.user-access]
- [metabase-enterprise.remote-sync.source :as source]
[metabase.actions.core :as actions]
[metabase.api.macros.defendpoint.closed-schemas :as closed-schemas]
[metabase.lib.core :as lib]
@@ -25,36 +24,15 @@
(t2/insert! :model/DataApp
:name "demo"
:display_name "Demo"
- :bundle_path "data_apps/demo/index.js"
- :bundle (.getBytes "BUNDLE" "UTF-8")
- :bundle_hash "abc123"))
+ :bundle_path "index.js"
+ :bundle (.getBytes "BUNDLE" "UTF-8")))
-(def ^:private fake-sha "0123456789abcdef0123456789abcdef01234567")
-
-(defn- snapshot
- "Build a snapshot (as the remote-sync import passes one) from a path->content
- map. `read-file` returns file text (a string) or nil; `list-dir` reuses the
- derivation the real non-git snapshots use, so the fake can't drift from it."
- [path->content & {:keys [sha] :or {sha fake-sha}}]
- {:sha sha
- :list-dir (fn [dir] (source/paths->children (keys path->content) dir))
- :read-file (fn [p] (get path->content p))})
-
-(defn- app-config
- "Render a per-app data_app.yaml from `{:name :path :allowed_hosts}`. No slug: an
- app's slug is the name of the directory the config sits in."
- [{:keys [name path allowed_hosts]}]
- (str (format "name: %s\npath: %s\n" name path)
- (when (seq allowed_hosts)
- (apply str "allowed_hosts:\n"
- (map #(format " - %s\n" %) allowed_hosts)))))
-
-(defn- app-files
- "Repo files for one data app under `data_apps/<dir>/`: its data_app.yaml plus a
- bundle at `path` with `bundle` content."
- [dir {:keys [path bundle] :as cfg}]
- {(format "data_apps/%s/data_app.yaml" dir) (app-config cfg)
- (format "data_apps/%s/%s" dir path) bundle})
+(def ^:private app-request
+ {:name "demo"
+ :display_name "Demo app"
+ :bundle_path "dist/index.js"
+ :allowed_hosts ["https://api.example.com"]
+ :bundle "DEMOBUNDLE"})
;;; ---------------------------------------------- Permissions ----------------------------------------------
@@ -93,8 +71,7 @@
(mt/with-premium-features #{:data-apps}
(mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
(create-app!)
- (is (nil? (t2/select-one-fn :resource_collection_id :model/DataApp :name "demo"))
- "precondition: the app has no resource collection")
+ (t2/update! :model/DataApp :name "demo" {:resource_collection_id nil})
(testing "an app with no resource collection is not published: neither its metadata nor its
bundle is served — to anyone — and a 409 lets the client show a \"not published\" screen"
(doseq [user [:rasta :crowberto]]
@@ -380,7 +357,7 @@
(deftest superuser-can-store-table-dependencies-test
(mt/with-premium-features #{:data-apps}
- (mt/with-model-cleanup [:model/DataApp]
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
(create-app!)
(let [table-ids [(mt/id :venues) (mt/id :orders)]]
(is (= (sort table-ids)
@@ -396,7 +373,7 @@
(deftest non-superuser-cannot-store-table-dependencies-test
(mt/with-premium-features #{:data-apps}
- (mt/with-model-cleanup [:model/DataApp]
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
(create-app!)
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :put 403 "apps/demo/table-dependencies"
@@ -404,7 +381,7 @@
(deftest table-dependencies-validates-table-ids-test
(mt/with-premium-features #{:data-apps}
- (mt/with-model-cleanup [:model/DataApp]
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
(create-app!)
(is (= "One or more tables do not exist."
(mt/user-http-request :crowberto :put 400 "apps/demo/table-dependencies"
@@ -555,7 +532,7 @@
(deftest user-permission-warnings-validates-users-test
(mt/with-premium-features #{:data-apps :tenants}
- (mt/with-model-cleanup [:model/DataApp]
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
(create-app!)
(testing "unknown users"
(mt/user-http-request :crowberto :post 404 "apps/demo/user-permission-warnings"
@@ -570,7 +547,7 @@
(deftest non-superuser-cannot-read-user-permission-warnings-test
(mt/with-premium-features #{:data-apps}
- (mt/with-model-cleanup [:model/DataApp]
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
(create-app!)
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403 "apps/demo/user-permission-warnings"
@@ -630,14 +607,14 @@
(deftest query-definition-must-use-a-table-source-test
(mt/with-premium-features #{:data-apps}
- (mt/with-model-cleanup [:model/DataApp]
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
(create-app!)
(is (some? (mt/user-http-request :crowberto :post 400 "apps/demo/query"
{:stages [{:source {:type "card" :id 1}}]}))))))
(deftest query-definition-source-must-be-valid-test
(mt/with-premium-features #{:data-apps}
- (mt/with-model-cleanup [:model/DataApp]
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
(create-app!)
(is (some? (mt/user-http-request :crowberto :post 400 "apps/demo/query"
{:stages [{:source {:type 1 :id (mt/id :venues)}}]}))))))
@@ -662,7 +639,7 @@
(is (= (select-keys first-response [:resource_collection_id :permission_group_id])
(select-keys second-response [:resource_collection_id :permission_group_id])))
(is (=? {:bundle nil :draft true}
- (t2/select-one :model/DataApp :name "draft-app")))))))
+ (t2/select-one [:model/DataApp :bundle :draft] :name "draft-app")))))))
(deftest non-superuser-cannot-create-a-data-app-draft-test
(mt/with-premium-features #{:data-apps}
@@ -673,9 +650,9 @@
(deftest data-app-draft-must-have-a-valid-slug-test
(mt/with-premium-features #{:data-apps}
- (mt/with-model-cleanup [:model/DataApp]
- (is (= "Data app draft slugs must use lowercase letters, numbers, and dashes."
- (mt/user-http-request :crowberto :post 400 "apps/Draft/draft")))
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (is (=? {:errors {:slug #".*lowercase letters.*"}}
+ (mt/user-http-request :crowberto :post 400 "apps/Draft/draft")))
(is (not (t2/exists? :model/DataApp :name "Draft"))))))
(deftest data-app-group-reaches-only-copied-actions-test
@@ -686,7 +663,7 @@
;; Its own slug: the `Data App: <slug>` group outlives other tests in
;; this namespace, so sharing "demo" collides on the group name.
(let [{:keys [permission_group_id resource_collection_id]}
- (data-app.sync/ensure-draft! "action-perms-app")
+ (data-apps.apps/ensure-draft! "action-perms-app")
metadata-provider (mt/metadata-provider)
venues (lib.metadata/table metadata-provider (mt/id :venues))]
(perms/add-user-to-group! (mt/user->id :rasta) permission_group_id)
@@ -722,8 +699,8 @@
(t2/insert! :model/DataApp :name "ready" :display_name "Ready" :bundle_path "data_apps/ready/index.js")
(t2/insert! :model/DataApp :name "disabled" :display_name "Disabled" :bundle_path "data_apps/disabled/index.js"
:enabled false)
- (t2/insert! :model/DataApp :name "failed" :display_name "Failed" :bundle_path "data_apps/failed/index.js"
- :sync_error "Could not read bundle")
+ (t2/insert! :model/DataApp :name "draft" :display_name "Draft" :bundle_path "data_apps/draft/index.js"
+ :draft true)
(is (=? [{:name "ready" :display_name "Ready"}]
(mt/user-http-request :rasta :get 200 "apps?available=true"))))))
@@ -820,50 +797,75 @@
(get-in by-name ["withhosts" :allowed_hosts])))
(is (= [] (get-in by-name ["nohosts" :allowed_hosts]))))))))
-;;; ----------------------------------------------------- Sync -----------------------------------------------------
-
-(deftest import-materializes-apps-test
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [result (data-app.sync/import-from-snapshot!
- (snapshot (merge (app-files "sales" {:name "Sales" :path "dist/index.js" :bundle "SALES-BUNDLE"})
- (app-files "ops" {:name "Ops" :path "dist/app.js" :bundle "OPS-BUNDLE"}))))]
- (is (=? {:synced 2} result))
- (is (= #{"sales" "ops"} (t2/select-fn-set :name :model/DataApp)))
- (let [sales (t2/select-one :model/DataApp :name "sales")]
- (is (= "SALES-BUNDLE" (String. ^bytes (:bundle sales) "UTF-8")))
- (is (= "Sales" (:display_name sales)))
- (is (= "data_apps/sales/dist/index.js" (:bundle_path sales)))
- (is (true? (:enabled sales)))
- (is (= fake-sha (:last_synced_sha sales)))
- (is (nil? (:sync_error sales)))))))
+;;; ------------------------------------------------- Create & update -------------------------------------------------
-(deftest import-stores-allowed-hosts-test
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (testing "allowed_hosts from data_app.yaml are persisted on the row"
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "sales" {:name "Sales" :path "dist/index.js" :bundle "B"
- :allowed_hosts ["https://api.example.com" "https://*.acme.com"]})))
- (is (= ["https://api.example.com" "https://*.acme.com"]
- (:allowed_hosts (t2/select-one :model/DataApp :name "sales")))))
- (testing "re-syncing without allowed_hosts clears them to an empty list"
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "sales" {:name "Sales" :path "dist/index.js" :bundle "B"})))
- (is (= [] (:allowed_hosts (t2/select-one :model/DataApp :name "sales")))))))
+(deftest create-endpoint-test
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{:data-apps}
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (testing "a non-superuser cannot create an app"
+ (is (= "You don't have permissions to do that."
+ (mt/user-http-request :rasta :post 403 "apps" app-request))))
+ (testing "a superuser creates an app with its manifest fields and bundle"
+ (is (=? {:name "demo"
+ :display_name "Demo app"
+ :bundle_path "dist/index.js"
+ :allowed_hosts ["https://api.example.com"]
+ :version 1
+ :enabled true
+ :draft false
+ :bundle_hash string?
+ :resource_collection_id pos-int?
+ :permission_group_id pos-int?}
+ (mt/user-http-request :crowberto :post 200 "apps"
+ (assoc app-request :bundle_path "./dist/index.js"))))
+ (is (str/includes? (str (mt/user-real-request :crowberto :get 200 "apps/demo/bundle"))
+ "DEMOBUNDLE")))
+ (testing "a taken slug is refused"
+ (is (= "A data app with this slug already exists."
+ (mt/user-http-request :crowberto :post 409 "apps" app-request))))
+ (testing "invalid manifest fields are refused"
+ (doseq [bad [{:name "Not A Slug"}
+ {:name "repo-status"}
+ {:bundle_path "../escape.js"}
+ {:allowed_hosts ["ftp://example.com"]}
+ {:version 0}]]
+ (mt/user-http-request :crowberto :post 400 "apps" (merge app-request {:name "other"} bad))
+ (is (not (t2/exists? :model/DataApp :name "other")) (str "should refuse: " (pr-str bad)))))))))
+
+(deftest create-endpoint-fills-a-draft-test
+ (mt/with-premium-features #{:data-apps}
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (let [draft (mt/user-http-request :crowberto :post 200 "apps/demo/draft")]
+ (is (=? {:id (:id draft)
+ :draft false
+ :resource_collection_id (:resource_collection_id draft)}
+ (mt/user-http-request :crowberto :post 200 "apps" app-request)))))))
-(deftest import-prunes-apps-absent-from-snapshot-test
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (data-app.sync/import-from-snapshot!
- (snapshot (merge (app-files "keep" {:name "Keep" :path "index.js" :bundle "KEEP"})
- (app-files "gone" {:name "Gone" :path "index.js" :bundle "GONE"}))))
- (is (= #{"keep" "gone"} (t2/select-fn-set :name :model/DataApp)))
- ;; The connected repo is the source of truth: an app whose directory is gone
- ;; from a later snapshot is pruned. (An admin can also remove one explicitly
- ;; via DELETE /api/apps/:slug.)
- (is (=? {:removed 1}
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "keep" {:name "Keep" :path "index.js" :bundle "KEEP"})))))
- (is (= #{"keep"} (t2/select-fn-set :name :model/DataApp))
- "the app absent from the later snapshot is pruned")))
+(deftest update-endpoint-test
+ (mt/test-helpers-set-global-values!
+ (mt/with-premium-features #{:data-apps}
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (let [{:keys [bundle_hash]} (mt/user-http-request :crowberto :post 200 "apps" app-request)]
+ (testing "a non-superuser cannot update an app"
+ (is (= "You don't have permissions to do that."
+ (mt/user-http-request :rasta :put 403 "apps/demo" {:display_name "Mine"}))))
+ (testing "a superuser updates manifest fields and the bundle"
+ (let [updated (mt/user-http-request :crowberto :put 200 "apps/demo"
+ {:display_name "Renamed"
+ :description " What it does "
+ :allowed_hosts []
+ :bundle "NEWBUNDLE"})]
+ (is (=? {:display_name "Renamed" :description "What it does" :allowed_hosts []}
+ updated))
+ (is (not= bundle_hash (:bundle_hash updated)))
+ (is (str/includes? (str (mt/user-real-request :crowberto :get 200 "apps/demo/bundle"))
+ "NEWBUNDLE"))))
+ (testing "invalid manifest fields are refused"
+ (mt/user-http-request :crowberto :put 400 "apps/demo" {:bundle_path "/abs.js"})
+ (is (= "dist/index.js" (t2/select-one-fn :bundle_path :model/DataApp :name "demo"))))
+ (testing "updating a missing app 404s"
+ (mt/user-http-request :crowberto :put 404 "apps/missing" {:enabled false})))))))
(deftest delete-endpoint-test
(mt/test-helpers-set-global-values!
@@ -891,83 +893,16 @@
(testing "removing a non-existent app 404s"
(mt/user-http-request :crowberto :delete 404 "apps/missing"))))))))
-(deftest import-preserves-enabled-across-syncs-test
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "a" {:name "A" :path "index.js" :bundle "V1"})))
- (t2/update! :model/DataApp :name "a" {:enabled false})
- ;; a new bundle must not flip the admin's enabled toggle back on
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "a" {:name "A" :path "index.js" :bundle "V2"})))
- (let [a (t2/select-one :model/DataApp :name "a")]
- (is (false? (:enabled a)) "the disabled toggle is preserved")
- (is (= "V2" (String. ^bytes (:bundle a) "UTF-8")) "the bundle is still updated"))))
-
-(deftest import-per-app-error-test
- (testing "a missing bundle file fails just that app, not the whole import"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (data-app.sync/import-from-snapshot!
- (snapshot (merge (app-files "good" {:name "Good" :path "index.js" :bundle "GOOD"})
- ;; "bad" declares a path that doesn't exist
- {"data_apps/bad/data_app.yaml" (app-config {:name "Bad" :path "missing.js"})})))
- (is (= #{"good" "bad"} (t2/select-fn-set :name :model/DataApp)))
- (let [good (t2/select-one :model/DataApp :name "good")
- bad (t2/select-one :model/DataApp :name "bad")]
- (is (= "GOOD" (String. ^bytes (:bundle good) "UTF-8")))
- (is (nil? (:sync_error good)))
- (is (nil? (:bundle bad)))
- (is (str/includes? (:sync_error bad) "missing.js"))))))
-
-(deftest import-serves-each-app-from-its-directory-test
- (testing "the directory an app lives in is the slug it's served at — two apps can't collide on one"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (data-app.sync/import-from-snapshot!
- (snapshot (merge (app-files "one" {:name "One" :path "a.js" :bundle "A"})
- (app-files "two" {:name "Two" :path "b.js" :bundle "B"}))))
- (is (= #{"one" "two"} (t2/select-fn-set :name :model/DataApp))))))
-
-(deftest import-isolates-bad-config-test
- (testing "a malformed data_app.yaml is isolated: sibling apps in the same repo still sync and are not pruned, the bad one is reported"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "existing" {:name "Existing" :path "i.js" :bundle "E"})))
- ;; The repo still holds "existing" and adds "good", alongside a broken "bad".
- ;; The broken config must not abort the others, nor prune its siblings.
- (let [result (data-app.sync/import-from-snapshot!
- (snapshot (merge (app-files "existing" {:name "Existing" :path "i.js" :bundle "E"})
- (app-files "good" {:name "Good" :path "i.js" :bundle "GOOD"})
- {"data_apps/bad/data_app.yaml" "name: [unterminated"})))]
- (is (=? {:synced 2 :removed 0} result))
- (is (= 1 (count (:config-errors result))))
- (is (= #{"existing" "good"} (t2/select-fn-set :name :model/DataApp))
- "the bad config neither aborts nor prunes its sibling apps, and doesn't materialize itself")))))
-
-(deftest sync-from-snapshot!-never-throws-test
- (mt/with-premium-features #{:data-apps}
- (testing "a malformed data_app.yaml is isolated into :config-errors; the app just doesn't appear, the sync doesn't throw"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [result (data-app.sync/sync-from-snapshot!
- (snapshot {"data_apps/x/data_app.yaml" "name: [unterminated"}))]
- (is (seq (:config-errors result)))
- (is (empty? (t2/select-fn-set :name :model/DataApp))))))
- (testing "a clean sync materializes the app with no config errors"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [result (data-app.sync/sync-from-snapshot!
- (snapshot (app-files "a" {:name "A" :path "index.js" :bundle "A"})))]
- (is (empty? (:config-errors result)))
- (is (= #{"a"} (t2/select-fn-set :name :model/DataApp))))))))
-
;;; ----------------------------------------------------- API -----------------------------------------------------
(deftest list-and-bundle-endpoints-test
(mt/test-helpers-set-global-values!
(mt/with-premium-features #{:data-apps}
(mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "demo" {:name "Demo app" :path "dist/index.js" :bundle "DEMOBUNDLE"})))
- (testing "GET / lists the synced apps"
+ (mt/user-http-request :crowberto :post 200 "apps" app-request)
+ (testing "GET / lists the apps"
(is (=? [{:name "demo" :display_name "Demo app"
- :bundle_path "data_apps/demo/dist/index.js" :enabled true}]
+ :bundle_path "dist/index.js" :enabled true}]
(mt/user-http-request :crowberto :get 200 "apps"))))
(testing "GET /:slug/bundle serves the cached bytes"
(is (str/includes?
@@ -977,11 +912,11 @@
(deftest repo-status-endpoint-test
(mt/with-premium-features #{:data-apps}
(testing "reports no repository when none is connected"
- (mt/with-dynamic-fn-redefs [data-app.sync/repo-url (constantly nil)]
+ (mt/with-dynamic-fn-redefs [data-apps.apps/repo-url (constantly nil)]
(is (=? {:configured false :url nil}
(mt/user-http-request :crowberto :get 200 "apps/repo-status")))))
(testing "reports the connected repository URL"
- (mt/with-dynamic-fn-redefs [data-app.sync/repo-url (constantly "https://github.com/metabase/stats-remote-sync")]
+ (mt/with-dynamic-fn-redefs [data-apps.apps/repo-url (constantly "https://github.com/metabase/stats-remote-sync")]
(is (=? {:configured true :url "https://github.com/metabase/stats-remote-sync"}
(mt/user-http-request :crowberto :get 200 "apps/repo-status")))))))
@@ -989,8 +924,7 @@
(mt/test-helpers-set-global-values!
(mt/with-premium-features #{:data-apps}
(mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "demo" {:name "Demo" :path "index.js" :bundle "BUNDLE"})))
+ (mt/user-http-request :crowberto :post 200 "apps" app-request)
(testing "PUT /:slug can disable an app"
(is (=? {:name "demo" :enabled false}
(mt/user-http-request :crowberto :put 200 "apps/demo" {:enabled false}))))
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/apps_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/apps_test.clj
new file mode 100644
index 00000000000..f8e0c8d89ce
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/apps_test.clj
@@ -0,0 +1,37 @@
+(ns metabase-enterprise.data-apps.apps-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.data-apps.apps :as data-apps.apps]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private app-row
+ {:name "sales"
+ :display_name "Sales"
+ :bundle_path "dist/index.js"
+ :bundle (.getBytes "BUNDLE" "UTF-8")})
+
+(deftest concurrent-draft-creation-is-safe-test
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (let [drafts (doall (repeatedly 2 #(future (data-apps.apps/ensure-draft! "draft-app"))))]
+ (doseq [draft drafts]
+ @draft)
+ (is (= 1 (t2/count :model/DataApp :name "draft-app"))))))
+
+(deftest create-app-fills-a-draft-with-the-same-slug-test
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (let [{:keys [resource_collection_id]} (data-apps.apps/ensure-draft! "sales")
+ draft-id (t2/select-one-pk :model/DataApp :name "sales")]
+ (is (= draft-id (data-apps.apps/create-app! app-row)))
+ (is (=? {:display_name "Sales"
+ :draft false
+ :resource_collection_id resource_collection_id}
+ (t2/select-one :model/DataApp draft-id))))))
+
+(deftest create-app-refuses-a-taken-slug-test
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (data-apps.apps/create-app! app-row)
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"already exists"
+ (data-apps.apps/create-app! app-row)))))
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/config_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/config_test.clj
deleted file mode 100644
index 44c8e8f31b3..00000000000
--- a/enterprise/backend/test/metabase_enterprise/data_apps/config_test.clj
+++ /dev/null
@@ -1,146 +0,0 @@
-(ns metabase-enterprise.data-apps.config-test
- (:require
- [clojure.java.io :as io]
- [clojure.test :refer :all]
- [metabase-enterprise.data-apps.config :as data-app.config]))
-
-(set! *warn-on-reflection* true)
-
-(defn- ->bytes ^bytes [^String s]
- (.getBytes s "UTF-8"))
-
-(defn- parse
- "Parse a `data_app.yaml` as if it sat in `dir` (default `data_apps/sales`, so the
- parsed slug is `sales` — the slug is the directory's name, never a config key)."
- ([s] (parse s "data_apps/sales"))
- ([s dir] (data-app.config/parse-app-config (->bytes s) dir)))
-
-(deftest parse-valid-config-test
- (is (= {:slug "sales" :display_name "Sales dashboard" :description nil :version 1
- :path "dist/index.js" :allowed_hosts []}
- (parse "name: Sales dashboard
-path: ./dist/index.js"))))
-
-(deftest parse-version-test
- (testing "absent means 1: every app predates the field"
- (is (= 1 (:version (parse "name: X\npath: dist/index.js")))))
- (testing "a whole number is carried through"
- (is (= 3 (:version (parse "name: X\nversion: 3\npath: dist/index.js")))))
- (testing "anything but a positive whole number is rejected, not coerced or compared as-is"
- (doseq [bad ["0" "-1" "1.5" "'1'" "one" "[1]" "1.0.0"]]
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"version.*positive whole number"
- (parse (str "name: X\nversion: " bad "\npath: dist/index.js")))
- (str "should reject: " (pr-str bad))))))
-
-(deftest template-manifest-declares-the-supported-version-test
- (testing "the scaffolding template stamps the version this Metabase serves, so a new app is never born outdated"
- (is (= data-app.config/supported-app-version
- (:version (parse (slurp "skills/metabase-data-app-setup/template/data_app.yaml")))))))
-
-(deftest migration-upgrade-guides-cover-every-version-test
- (testing "the migrate skill ships one guide per upgrade up to the supported version, so an app at any older version has a path"
- (let [upgrades (->> (.listFiles (io/file "skills/metabase-data-app-migrate/references/upgrades"))
- (keep #(re-matches #"v(\d+)-to-v(\d+)\.md" (.getName ^java.io.File %)))
- (map (fn [[_ from to]] [(parse-long from) (parse-long to)]))
- sort)]
- (is (= (map (fn [n] [n (inc n)]) (range 1 data-app.config/supported-app-version))
- upgrades)))))
-
-(deftest outdated-test
- (testing "an app below the supported version is outdated; one at it is not"
- (with-redefs [data-app.config/supported-app-version 2]
- (is (data-app.config/outdated? {:version 1}))
- (is (not (data-app.config/outdated? {:version 2}))))))
-
-(deftest parse-description-test
- (testing "an optional one-liner is trimmed and carried through"
- (is (= "Pipeline health by region"
- (:description (parse "name: X\ndescription: ' Pipeline health by region '\npath: dist/index.js")))))
- (testing "absent → nil (the field is optional)"
- (is (nil? (:description (parse "name: X\npath: dist/index.js")))))
- (testing "blank → nil, so an unfilled placeholder doesn't become an empty description"
- (doseq [blank ["" "' '"]]
- (is (nil? (:description (parse (str "name: X\ndescription: " blank "\npath: dist/index.js"))))
- (str "should be nil for: " (pr-str blank)))))
- (testing "a YAML block scalar spanning several lines is folded onto one"
- (is (= "First line second line"
- (:description (parse "name: X\ndescription: |\n First line\n second line\npath: dist/index.js")))))
- (testing "a description at the length cap is accepted"
- (is (= 255
- (count (:description (parse (str "name: X\ndescription: " (apply str (repeat 255 "x"))
- "\npath: dist/index.js")))))))
- (testing "one over the cap is rejected — a paragraph here would be stored in full and returned on every list request"
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"description.*255 characters or fewer"
- (parse (str "name: X\ndescription: " (apply str (repeat 256 "x"))
- "\npath: dist/index.js")))))
- (testing "the cap applies to the folded value, not the raw YAML"
- (is (= 255
- (count (:description (parse (str "name: X\ndescription: >\n " (apply str (repeat 255 "x"))
- "\npath: dist/index.js")))))))
- (testing "a non-string description (a YAML list or mapping) is rejected, not str-ified into the row"
- (doseq [bad ["description: [foo, bar]"
- "description:\n key: value"]]
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"description.*must be a one-line string"
- (parse (str "name: X\n" bad "\npath: dist/index.js")))
- (str "should reject: " (pr-str bad))))))
-
-(deftest slug-comes-from-the-directory-test
- (testing "the app's slug is the name of the directory it lives in"
- (is (= "inventory" (:slug (parse "name: X\npath: dist/index.js" "data_apps/inventory")))))
- (testing "a directory name that isn't a usable slug is rejected — the app would have no URL to be served at"
- (doseq [bad ["Sales" "my_app" "sales app" "-sales" "sales-"]]
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"lowercase letters, numbers, and dashes"
- (parse "name: X\npath: dist/index.js" (str "data_apps/" bad)))
- (str "should reject the directory: " bad)))))
-
-(deftest parse-allowed-hosts-test
- (testing "valid entries are lowercased, trailing-slash-stripped, and de-duplicated"
- (is (= ["https://api.example.com" "https://*.internal.acme.com"]
- (:allowed_hosts
- (parse "name: X\npath: dist/index.js\nallowed_hosts:\n - https://API.example.com/\n - https://*.internal.acme.com\n - https://api.example.com")))))
- (testing "absent → empty vector"
- (is (= [] (:allowed_hosts (parse "name: X\npath: dist/index.js")))))
- (testing "a non-list value is rejected"
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"allowed_hosts.*must be a list"
- (parse "name: X\npath: dist/index.js\nallowed_hosts: https://api.example.com"))))
- (testing "invalid entries (bare *, path, non-http scheme, no scheme) are rejected"
- (doseq [bad ["*" "https://example.com/path" "ftp://example.com" "example.com"]]
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"not a valid allowed_hosts entry"
- (parse (str "name: X\npath: dist/index.js\nallowed_hosts:\n - \"" bad "\"")))
- (str "should reject: " bad)))))
-
-(deftest parse-strips-leading-dot-slash-test
- (is (= "dist/app.js"
- (:path (parse "name: X\npath: ./dist/app.js")))))
-
-(deftest unknown-fields-are-ignored-test
- (testing "unknown keys don't fail the parse — including a stray `slug`, which the directory name overrides"
- (is (= {:slug "sales" :display_name "X" :description nil :version 1 :path "dist/index.js" :allowed_hosts []}
- (parse "name: X\nslug: elsewhere\nfuture_option: 1\npath: dist/index.js")))))
-
-(deftest parse-errors-test
- (testing "missing name"
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"name"
- (parse "path: dist/index.js"))))
- (testing "missing path"
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"path"
- (parse "name: X"))))
- (testing "malformed yaml"
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Could not parse"
- (parse "name: [unterminated"))))
- (testing "path traversal is rejected"
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"must not contain"
- (parse "name: X\npath: ../other/dist/index.js")))
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"must not contain"
- (parse "name: X\npath: dist/../../escape.js"))))
- (testing "a directory whose name collides with an API sub-route is rejected"
- (doseq [slug ["repo-status" "sandbox-host"]]
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"reserved slug"
- (parse "name: X\npath: dist/index.js" (str "data_apps/" slug)))))))
-
-(deftest parse-errors-carry-400-test
- (try
- (parse "name: X")
- (is false "should have thrown")
- (catch clojure.lang.ExceptionInfo e
- (is (= 400 (:status-code (ex-data e)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/models/data_app_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/models/data_app_test.clj
index 516c1b090d5..5a14a67a634 100644
--- a/enterprise/backend/test/metabase_enterprise/data_apps/models/data_app_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/models/data_app_test.clj
@@ -1,7 +1,6 @@
(ns metabase-enterprise.data-apps.models.data-app-test
- "Unit coverage for the DataApp model: the blob-excluding reads, JSON that never
- leaks the bundle bytes, the `allowed_hosts` NULL→[] read coercion, and the
- superuser-only permission gating."
+ "Unit coverage for the DataApp model: the write hooks' invariants, the blob-excluding reads, JSON that never
+ leaks the bundle bytes, the `allowed_hosts` NULL→[] read coercion, and the superuser-only permission gating."
(:require
[clojure.test :refer :all]
[metabase-enterprise.data-apps.db :as data-apps.db]
@@ -19,32 +18,72 @@
:model/DataApp
(merge {:name "m"
:display_name "M"
- :bundle_path "data_apps/m/index.js"
- :bundle (.getBytes "BUNDLEBYTES" "UTF-8")
- :bundle_hash "hash"}
+ :bundle_path "index.js"
+ :bundle (.getBytes "BUNDLEBYTES" "UTF-8")}
extra)))
+(deftest bundle-hash-follows-the-bundle-test
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (let [id (insert-app! :bundle_hash "stale")
+ hash #(t2/select-one-fn :bundle_hash :model/DataApp id)
+ v1 (hash)]
+ (is (= 64 (count v1)) "the hash is computed from the bundle, not taken from the row")
+ (t2/update! :model/DataApp id {:display_name "Renamed"})
+ (is (= v1 (hash)) "a change that leaves the bundle alone keeps its hash")
+ (t2/update! :model/DataApp id {:display_name "Renamed"})
+ (is (= v1 (hash)) "an update that changes nothing is a no-op")
+ (t2/update! :model/DataApp id {:bundle (.getBytes "OTHER" "UTF-8")})
+ (is (not= v1 (hash)))
+ (t2/update! :model/DataApp id {:bundle nil})
+ (is (nil? (hash))))))
+
+(deftest insert-creates-the-resources-the-app-owns-test
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (let [{:keys [resource_collection_id permission_group_id]} (t2/select-one :model/DataApp (insert-app!))]
+ (is (t2/exists? :model/Collection :id resource_collection_id))
+ (is (t2/exists? :model/PermissionsGroup :id permission_group_id :is_data_app_group true)))))
+
+(deftest writes-are-normalized-and-validated-against-the-schema-test
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (let [id (insert-app! :display_name " M " :bundle_path "./dist/index.js"
+ :allowed_hosts ["https://API.example.com/"])]
+ (is (=? {:display_name "M" :bundle_path "dist/index.js" :version 1 :allowed_hosts ["https://api.example.com"]}
+ (t2/select-one :model/DataApp id)))
+ (testing "an update with an invalid column is refused"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"does not match schema"
+ (t2/update! :model/DataApp id {:bundle_path "../escape.js"})))
+ (is (= "dist/index.js" (t2/select-one-fn :bundle_path :model/DataApp id)))))
+ (testing "an insert without a required manifest column is refused"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"does not match schema"
+ (t2/insert! :model/DataApp {:name "x" :bundle_path "i.js"}))))
+ (testing "an insert with an invalid slug is refused"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"does not match schema"
+ (insert-app! :name "Not A Slug"))))))
+
(deftest to-json-never-includes-the-bundle-bytes-test
- (mt/with-model-cleanup [:model/DataApp]
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
(insert-app!)
- (let [app (t2/select-one :model/DataApp :name "m")
+ (let [app (t2/select-one [:model/DataApp :bundle :display_name] :name "m")
decoded (json/decode (json/encode app))]
- (is (contains? app :bundle) "the selected instance still carries the raw bundle")
+ (is (contains? app :bundle) "the instance selected with its bundle carries the raw bytes")
(is (not (contains? decoded "bundle"))
"but the JSON representation omits it")
(is (= "M" (get decoded "display_name"))))))
-(deftest non-blob-selects-exclude-the-bundle-test
- (mt/with-model-cleanup [:model/DataApp]
+(deftest default-selects-exclude-the-bundle-test
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
(insert-app! :allowed_hosts ["https://api.example.com"])
- (testing "non-blob-data-app-by-slug returns metadata without the bundle blob"
- (let [app (data-apps.db/non-blob-data-app-by-slug "m")]
+ (is (not (contains? (t2/select-one :model/DataApp :name "m") :bundle)))
+ (is (= "BUNDLEBYTES" (String. ^bytes (data-apps.db/data-app-bundle (t2/select-one-pk :model/DataApp :name "m"))
+ "UTF-8")))
+ (testing "data-app-by-slug returns metadata without the bundle blob"
+ (let [app (data-apps.db/data-app-by-slug "m")]
(is (not (contains? app :bundle)))
(is (= "M" (:display_name app)))
(is (= ["https://api.example.com"] (:allowed_hosts app)))))))
(deftest allowed-hosts-reads-as-a-vector-never-nil-test
- (mt/with-model-cleanup [:model/DataApp]
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
(testing "a row stored with NULL allowed_hosts reads back as []"
(insert-app!)
(is (= [] (:allowed_hosts (t2/select-one :model/DataApp :name "m")))))
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/schema_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/schema_test.clj
new file mode 100644
index 00000000000..0a5678d4cef
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/schema_test.clj
@@ -0,0 +1,111 @@
+(ns metabase-enterprise.data-apps.schema-test
+ (:require
+ [clojure.java.io :as io]
+ [clojure.test :refer :all]
+ [metabase-enterprise.data-apps.config :as data-app.config]
+ [metabase-enterprise.data-apps.schema :as data-apps.schema]
+ [metabase.lib.core :as lib]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.yaml :as yaml]))
+
+(set! *warn-on-reflection* true)
+
+(defn- normalize [schema x]
+ (lib/normalize schema x))
+
+(defn- valid? [schema x]
+ (mr/validate schema (normalize schema x)))
+
+(deftest data-app-update-normalizes-each-column-it-is-given-test
+ (is (= {:name "sales" :display_name "Sales dashboard" :description nil :version 1
+ :bundle_path "dist/index.js" :allowed_hosts []}
+ (normalize ::data-apps.schema/data-app.update
+ {:name "sales" :display_name " Sales dashboard " :description " " :version nil
+ :bundle_path "./dist/index.js" :allowed_hosts nil})))
+ (is (= {:enabled false} (normalize ::data-apps.schema/data-app.update {:enabled false}))))
+
+(deftest data-app-insert-requires-the-manifest-columns-test
+ (is (valid? ::data-apps.schema/data-app.insert {:name "a" :display_name "A" :bundle_path "i.js"}))
+ (doseq [k [:name :display_name :bundle_path]]
+ (is (not (valid? ::data-apps.schema/data-app.insert
+ (dissoc {:name "a" :display_name "A" :bundle_path "i.js"} k)))
+ (str "should require " k))))
+
+(deftest version-test
+ (testing "nil means 1: every app predates the field"
+ (is (= 1 (normalize ::data-apps.schema/version nil))))
+ (testing "anything but a positive whole number is rejected, not coerced"
+ (doseq [bad [0 -1 1.5 "1" "one" [1]]]
+ (is (not (valid? ::data-apps.schema/version bad)) (str "should reject: " (pr-str bad))))))
+
+(deftest template-manifest-declares-the-supported-version-test
+ (testing "the scaffolding template stamps the version this Metabase serves, so a new app is never born outdated"
+ (is (= data-app.config/supported-app-version
+ (:version (yaml/parse-string (slurp "skills/metabase-data-app-setup/template/data_app.yaml")))))))
+
+(deftest migration-upgrade-guides-cover-every-version-test
+ (testing "the migrate skill ships one guide per upgrade up to the supported version, so an app at any older version has a path"
+ (let [upgrades (->> (.listFiles (io/file "skills/metabase-data-app-migrate/references/upgrades"))
+ (keep #(re-matches #"v(\d+)-to-v(\d+)\.md" (.getName ^java.io.File %)))
+ (map (fn [[_ from to]] [(parse-long from) (parse-long to)]))
+ sort)]
+ (is (= (map (fn [n] [n (inc n)]) (range 1 data-app.config/supported-app-version))
+ upgrades)))))
+
+(deftest outdated-test
+ (testing "an app below the supported version is outdated; one at it is not"
+ (with-redefs [data-app.config/supported-app-version 2]
+ (is (data-app.config/outdated? {:version 1}))
+ (is (not (data-app.config/outdated? {:version 2}))))))
+
+(deftest description-test
+ (testing "an optional one-liner is trimmed, and a multi-line value folded onto one line"
+ (is (= "Pipeline health by region" (normalize ::data-apps.schema/description " Pipeline health by region ")))
+ (is (= "First line second line" (normalize ::data-apps.schema/description "First line\n second line\n"))))
+ (testing "blank → nil, so an unfilled placeholder doesn't become an empty description"
+ (doseq [blank [nil "" " "]]
+ (is (nil? (normalize ::data-apps.schema/description blank)) (str "should be nil for: " (pr-str blank)))))
+ (testing "the length cap applies to the folded value — a paragraph here would be returned on every list request"
+ (is (valid? ::data-apps.schema/description (apply str (repeat 255 "x"))))
+ (is (not (valid? ::data-apps.schema/description (apply str (repeat 256 "x"))))))
+ (testing "a non-string description (a YAML list or mapping) is rejected, not str-ified into the row"
+ (doseq [bad [["foo" "bar"] {:key "value"}]]
+ (is (not (valid? ::data-apps.schema/description bad)) (str "should reject: " (pr-str bad))))))
+
+(deftest slug-test
+ (testing "a slug is used verbatim"
+ (is (= "inventory-2" (normalize ::data-apps.schema/slug "inventory-2"))))
+ (testing "a slug that can't appear in a URL as-is, or collides with an API sub-route, is rejected"
+ (doseq [bad [nil "" "Sales" "my_app" "sales app" "-sales" "sales-" "sales\n" "repo-status" "sandbox-host"
+ (apply str (repeat 101 "a"))]]
+ (is (not (valid? ::data-apps.schema/slug bad)) (str "should reject: " (pr-str bad))))))
+
+(deftest display-name-test
+ (is (= "X" (normalize ::data-apps.schema/display-name " X ")))
+ (doseq [bad [nil "" " "]]
+ (is (not (valid? ::data-apps.schema/display-name bad)) (str "should reject: " (pr-str bad)))))
+
+(deftest allowed-hosts-test
+ (testing "valid entries are lowercased, trailing-slash-stripped, and de-duplicated"
+ (is (= ["https://api.example.com" "https://*.internal.acme.com"]
+ (normalize ::data-apps.schema/allowed-hosts
+ ["https://API.example.com/" "https://*.internal.acme.com" "https://api.example.com"]))))
+ (testing "nil → empty vector"
+ (is (= [] (normalize ::data-apps.schema/allowed-hosts nil))))
+ (testing "a non-list value is rejected"
+ (is (not (valid? ::data-apps.schema/allowed-hosts "https://api.example.com"))))
+ (testing "invalid entries (bare *, path, non-http scheme, no scheme) are rejected"
+ (doseq [bad ["*" "https://example.com/path" "ftp://example.com" "example.com" "https://a.com\nx"]]
+ (is (not (valid? ::data-apps.schema/allowed-hosts [bad])) (str "should reject: " bad)))))
+
+(deftest bundle-path-test
+ (testing "a leading ./ is dropped"
+ (is (= "dist/app.js" (normalize ::data-apps.schema/bundle-path "./dist/app.js"))))
+ (testing "a missing path, or one leaving the app's directory, is rejected"
+ (doseq [bad [nil "" " " "../other/dist/index.js" "dist/../../escape.js" "/etc/passwd" "dist//index.js"
+ "..\\..\\escape.js" "data_app.yaml"]]
+ (is (not (valid? ::data-apps.schema/bundle-path bad)) (str "should reject: " (pr-str bad))))))
+
+(deftest bundle-test
+ (is (valid? ::data-apps.schema/bundle (byte-array 3)))
+ (is (not (valid? ::data-apps.schema/bundle (byte-array (inc data-apps.schema/max-bundle-bytes))))))
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.clj
new file mode 100644
index 00000000000..9394bd9cdf4
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.clj
@@ -0,0 +1,179 @@
+(ns metabase-enterprise.data-apps.serialization-test
+ (:require
+ [clojure.java.io :as io]
+ [clojure.test :refer :all]
+ [metabase-enterprise.data-apps.apps :as data-apps.apps]
+ [metabase-enterprise.serialization.core :as serialization]
+ [metabase-enterprise.serialization.test-util :as ts]
+ [metabase-enterprise.serialization.v2.extract :as extract]
+ [metabase.models.serialization :as serdes]
+ [metabase.test :as mt]
+ [metabase.util.yaml :as yaml]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn- insert-app! [& {:as extra}]
+ (t2/insert-returning-instance! :model/DataApp
+ (merge {:name "sales-ops"
+ :display_name "Sales Ops"
+ :bundle_path "dist/index.js"
+ :bundle (.getBytes "console.log(1)" "UTF-8")
+ :allowed_hosts ["https://api.example.com"]}
+ extra)))
+
+(defn- export! [dir]
+ (serialization/store! (serdes/extract-all "DataApp" {}) (serialization/file-writer dir)))
+
+(defn- import! [dir]
+ (serialization/load-metabase! (serialization/ingest-yaml dir)))
+
+(defn- bundle-text [app]
+ (String. ^bytes (t2/select-one-fn :bundle [:model/DataApp :bundle] :id (:id app)) "UTF-8"))
+
+(defn- write-app-files!
+ "Write a serialized data app with `yaml` fields and the `files` next to it under `data_apps/<dir>/`."
+ [dump-dir dir yaml-fields files]
+ (let [app-dir (io/file dump-dir "data_apps" dir)]
+ (.mkdirs app-dir)
+ (spit (io/file app-dir "data_app.yaml") (yaml/generate-string yaml-fields))
+ (doseq [[path content] files
+ :let [f (io/file app-dir ^String path)]]
+ (io/make-parents f)
+ (spit f content))))
+
+(defn- app-yaml [entity-id slug & {:as extra}]
+ (merge {:serdes/meta [{:model "DataApp" :id entity-id :label slug}]
+ :entity_id entity-id
+ :slug slug
+ :name "Sales"
+ :path "dist/index.js"}
+ extra))
+
+(deftest export-writes-the-manifest-and-the-bundle-file-test
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (ts/with-random-dump-dir [dump-dir "data-app-export-"]
+ (let [app (insert-app! :description "Pipeline health")]
+ (export! dump-dir)
+ (testing "the manifest is a serdes YAML in the app's directory, keyed like a hand-written data_app.yaml"
+ (is (= {:serdes/meta [{:model "DataApp" :id (:entity_id app) :label "sales_ops"}]
+ :entity_id (:entity_id app)
+ :slug "sales-ops"
+ :name "Sales Ops"
+ :description "Pipeline health"
+ :path "dist/index.js"
+ :allowed_hosts ["https://api.example.com"]}
+ (dissoc (yaml/from-file (io/file dump-dir "data_apps" "sales-ops" "data_app.yaml"))
+ :created_at))))
+ (testing "the bundle is a plain file at its path next to the manifest"
+ (is (= "console.log(1)"
+ (slurp (io/file dump-dir "data_apps" "sales-ops" "dist" "index.js")))))))))
+
+(deftest drafts-are-not-exported-test
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (data-apps.apps/ensure-draft! "draft-app")
+ (is (empty? (into [] (serdes/extract-all "DataApp" {}))))))
+
+(deftest round-trip-test
+ (mt/with-premium-features #{:data-apps}
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (ts/with-random-dump-dir [dump-dir "data-app-round-trip-"]
+ (let [app (insert-app!)]
+ (export! dump-dir)
+ (t2/delete! :model/DataApp (:id app))
+ (import! dump-dir)
+ (let [imported (t2/select-one :model/DataApp :entity_id (:entity_id app))]
+ (is (=? {:name "sales-ops"
+ :display_name "Sales Ops"
+ :bundle_path "dist/index.js"
+ :bundle_hash (:bundle_hash app)
+ :allowed_hosts ["https://api.example.com"]
+ :draft false}
+ imported))
+ (is (= "console.log(1)" (bundle-text imported)))
+ (testing "the import creates the collection and permission group the app owns"
+ (is (t2/exists? :model/Collection :id (:resource_collection_id imported)))
+ (is (t2/exists? :model/PermissionsGroup :id (:permission_group_id imported))))))))))
+
+(deftest import-updates-in-place-and-keeps-local-state-test
+ (mt/with-premium-features #{:data-apps}
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (ts/with-random-dump-dir [dump-dir "data-app-update-"]
+ (let [app (insert-app! :enabled false)]
+ (write-app-files! dump-dir "sales-ops" (app-yaml (:entity_id app) "sales-ops" :name "Renamed"
+ :path "./dist/index.js")
+ {"dist/index.js" "console.log(2)"})
+ (import! dump-dir)
+ (let [updated (t2/select-one :model/DataApp :id (:id app))]
+ (is (=? {:display_name "Renamed"
+ :enabled false
+ :resource_collection_id (:resource_collection_id app)
+ :permission_group_id (:permission_group_id app)}
+ updated))
+ (is (= "console.log(2)" (bundle-text updated)))
+ (is (not= (:bundle_hash app) (:bundle_hash updated)))))))))
+
+(deftest import-takes-over-a-draft-with-the-same-slug-test
+ (mt/with-premium-features #{:data-apps}
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (ts/with-random-dump-dir [dump-dir "data-app-draft-"]
+ (let [draft (data-apps.apps/ensure-draft! "draft-app")
+ id (t2/select-one-pk :model/DataApp :name "draft-app")]
+ (write-app-files! dump-dir "draft_app" (app-yaml "Ld3cXiYs9n8HP3q3FvC7R" "draft-app")
+ {"dist/index.js" "BUNDLE"})
+ (import! dump-dir)
+ (is (=? {:id id
+ :entity_id "Ld3cXiYs9n8HP3q3FvC7R"
+ :draft false
+ :resource_collection_id (:resource_collection_id draft)}
+ (t2/select-one :model/DataApp :name "draft-app"))))))))
+
+(deftest import-rejects-invalid-apps-test
+ (mt/with-premium-features #{:data-apps}
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (testing "a missing bundle file"
+ (ts/with-random-dump-dir [dump-dir "data-app-missing-"]
+ (write-app-files! dump-dir "x" (app-yaml "pZrj7PDuz3vSWYYi0QFhd" "x") {})
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Failed to load"
+ (import! dump-dir)))))
+ (testing "a manifest field that fails validation"
+ (ts/with-random-dump-dir [dump-dir "data-app-invalid-"]
+ (write-app-files! dump-dir "x" (app-yaml "pZrj7PDuz3vSWYYi0QFhd" "Not A Slug")
+ {"dist/index.js" "BUNDLE"})
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Failed to load"
+ (import! dump-dir)))))
+ (testing "a bundle path leaving the app's directory is never read"
+ (ts/with-random-dump-dir [dump-dir "data-app-traversal-"]
+ (write-app-files! dump-dir "x" (app-yaml "pZrj7PDuz3vSWYYi0QFhd" "x" :path "../../secret.js") {})
+ (is (= "Invalid resource file path: ../../secret.js"
+ (try
+ (serialization/ingest-one (serialization/ingest-yaml dump-dir)
+ [{:model "DataApp" :id "pZrj7PDuz3vSWYYi0QFhd"}])
+ nil
+ (catch clojure.lang.ExceptionInfo e
+ (ex-message (ex-cause e))))))))
+ (is (not (t2/exists? :model/DataApp))))))
+
+(deftest import-clears-an-omitted-description-test
+ (mt/with-premium-features #{:data-apps}
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (ts/with-random-dump-dir [dump-dir "data-app-description-"]
+ (let [app (insert-app! :description "Pipeline health")]
+ (write-app-files! dump-dir "sales-ops" (app-yaml (:entity_id app) "sales-ops") {"dist/index.js" "B"})
+ (import! dump-dir)
+ (is (nil? (t2/select-one-fn :description :model/DataApp :id (:id app)))))))))
+
+(deftest import-does-not-take-over-an-app-that-is-not-a-draft-test
+ (mt/with-premium-features #{:data-apps}
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (ts/with-random-dump-dir [dump-dir "data-app-takeover-"]
+ (let [app (insert-app!)]
+ (write-app-files! dump-dir "sales-ops" (app-yaml "Ld3cXiYs9n8HP3q3FvC7R" "sales-ops") {"dist/index.js" "B"})
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Failed to load" (import! dump-dir)))
+ (is (=? {:entity_id (:entity_id app)} (t2/select-one :model/DataApp :id (:id app)))))))))
+
+(deftest export-includes-data-apps-test
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ (let [app (insert-app!)]
+ (is (some #(= [{:model "DataApp" :id (:entity_id app)}] (map (fn [m] (dissoc m :label)) (:serdes/meta %)))
+ (into [] (extract/extract {:no-collections true :no-data-model true :no-settings true})))))))
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/sync_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/sync_test.clj
deleted file mode 100644
index 97b73fa380b..00000000000
--- a/enterprise/backend/test/metabase_enterprise/data_apps/sync_test.clj
+++ /dev/null
@@ -1,386 +0,0 @@
-(ns metabase-enterprise.data-apps.sync-test
- "Unit coverage for the materialization edge cases not exercised through the API
- suite: the `:changed` accounting (content vs. sha/timestamp bumps) and the
- oversized-bundle guard (rejected with a sync_error, previously cached bundle
- retained)."
- (:require
- [clojure.string :as str]
- [clojure.test :refer :all]
- [metabase-enterprise.data-apps.db :as data-apps.db]
- [metabase-enterprise.data-apps.resources :as data-app.resources]
- [metabase-enterprise.data-apps.sync :as data-app.sync]
- [metabase-enterprise.remote-sync.source :as source]
- [metabase.permissions.core :as perms]
- [metabase.test :as mt]
- [toucan2.core :as t2]))
-
-(set! *warn-on-reflection* true)
-
-(def ^:private fake-sha "0123456789abcdef0123456789abcdef01234567")
-
-(defn- snapshot
- [path->content & {:keys [sha] :or {sha fake-sha}}]
- {:sha sha
- :list-dir (fn [dir] (source/paths->children (keys path->content) dir))
- :read-file (fn [p] (get path->content p))})
-
-(defn- app-files
- "The repo files for one app in `data_apps/<dir>`. `dir` is the app's slug — the
- config declares no slug, it is the directory's name."
- [dir {:keys [name path bundle description version]}]
- {(format "data_apps/%s/data_app.yaml" dir)
- (str (format "name: %s\npath: %s\n" name path)
- (when description (format "description: %s\n" description))
- (when version (format "version: %s\n" version)))
- (format "data_apps/%s/%s" dir path) bundle})
-
-(deftest sync-from-snapshot-is-gated-by-the-data-apps-feature-test
- (testing "the remote-sync entry point does nothing without the :data-apps feature —
- an instance then behaves exactly as if data apps did not exist"
- (let [files (app-files "a" {:name "A" :path "index.js" :bundle "V1"})]
- (mt/with-premium-features #{}
- (mt/with-model-cleanup [:model/DataApp]
- (is (nil? (data-app.sync/sync-from-snapshot! (snapshot files)))
- "returns nil rather than a sync result")
- (is (not (t2/exists? :model/DataApp :name "a"))
- "materializes no data app")))
- (mt/with-premium-features #{:data-apps}
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (is (=? {:synced 1 :changed 1}
- (data-app.sync/sync-from-snapshot! (snapshot files)))
- "with the feature it materializes the app as usual")
- (is (t2/exists? :model/DataApp :name "a")))))))
-
-(deftest resource-provisioning-failures-are-isolated-test
- (mt/with-model-cleanup [:model/DataApp]
- (let [files (merge (app-files "broken" {:name "Broken" :path "index.js" :bundle "BROKEN"})
- (app-files "working" {:name "Working" :path "index.js" :bundle "WORKING"}))]
- (mt/with-dynamic-fn-redefs [data-app.resources/ensure-resources!
- (fn [app]
- (when (= "broken" (:name app))
- (throw (ex-info "Resource provisioning failed." {}))))]
- (is (=? {:synced 2 :changed 2}
- (data-app.sync/import-from-snapshot! (snapshot files))))
- (is (= "Resource provisioning failed."
- (t2/select-one-fn :sync_error :model/DataApp :name "broken")))
- (is (nil? (t2/select-one-fn :sync_error :model/DataApp :name "working")))))))
-
-(deftest sync-creates-stable-permission-resources-test
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [database-id (mt/id)
- files (app-files "sales" {:name "Sales" :path "index.js" :bundle "V1"})]
- (data-app.sync/import-from-snapshot! (snapshot files))
- (let [{:keys [resource_collection_id permission_group_id]}
- (t2/select-one :model/DataApp :name "sales")]
- (testing "the first sync creates a dedicated group and collection"
- (is (pos-int? resource_collection_id))
- (is (pos-int? permission_group_id))
- (is (= "Data App: sales"
- (t2/select-one-fn :name :model/Collection :id resource_collection_id)))
- (is (= "Data App: sales"
- (t2/select-one-fn :name :model/PermissionsGroup :id permission_group_id)))
- (is (t2/exists? :model/Permissions
- :group_id permission_group_id
- :object (perms/collection-read-path resource_collection_id)))
- (is (not (t2/exists? :model/Permissions
- :group_id permission_group_id
- :object (perms/collection-readwrite-path resource_collection_id))))
- (let [permissions (t2/select-fn-vec :perm_value :model/DataPermissions
- :group_id permission_group_id
- :db_id database-id
- :perm_type :perms/create-queries)]
- (is (seq permissions))
- (is (every? #(= :no %) permissions))))
- (testing "later syncs reuse the same resources"
- (data-app.sync/import-from-snapshot! (snapshot files))
- (is (=? {:resource_collection_id resource_collection_id
- :permission_group_id permission_group_id}
- (t2/select-one :model/DataApp :name "sales"))))
- (testing "removing the app deletes its resources"
- (data-app.sync/import-from-snapshot! (snapshot {}))
- (is (not (t2/exists? :model/Collection :id resource_collection_id)))
- (is (not (t2/exists? :model/PermissionsGroup :id permission_group_id))))))))
-
-(deftest sync-restores-deleted-collection-and-permission-group-test
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [files (app-files "sales" {:name "Sales" :path "index.js" :bundle "V1"})]
- (data-app.sync/import-from-snapshot! (snapshot files))
- (let [before (select-keys (t2/select-one :model/DataApp :name "sales")
- [:resource_collection_id :permission_group_id])]
- (t2/delete! :model/Collection :id (:resource_collection_id before))
- (t2/delete! :model/PermissionsGroup :id (:permission_group_id before))
- (data-app.sync/import-from-snapshot! (snapshot files))
- ; after syncing the app again, the data app collection and
- ; permission group should be re-created
- (let [after (select-keys (t2/select-one :model/DataApp :name "sales") (keys before))]
- (is (every? pos-int? (vals after)))
- (is (t2/exists? :model/Collection :id (:resource_collection_id after)))
- (is (t2/exists? :model/PermissionsGroup :id (:permission_group_id after)))
- (is (not= before after)))))))
-
-(deftest changed-count-tracks-content-not-sha-bumps-test
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [files (app-files "a" {:name "A" :path "index.js" :bundle "V1"})]
- (testing "the first sync counts the new app"
- (is (=? {:synced 1 :changed 1}
- (data-app.sync/import-from-snapshot! (snapshot files)))))
- (testing "re-syncing identical content at a new sha is not a change"
- (is (=? {:synced 1 :changed 0}
- (data-app.sync/import-from-snapshot!
- (snapshot files :sha "ffffffffffffffffffffffffffffffffffffffff")))))
- (testing "a metadata-only change counts"
- (is (=? {:changed 1}
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "a" {:name "A renamed" :path "index.js" :bundle "V1"}))))))
- (testing "a bundle content change counts"
- (is (=? {:changed 1}
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "a" {:name "A renamed" :path "index.js" :bundle "V2"})))))))))
-
-(deftest description-is-optional-and-tracked-like-other-metadata-test
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [sync-app (fn [& {:as app}]
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "a" (merge {:name "A" :path "index.js" :bundle "V1"} app)))))]
- (testing "an app that declares no description syncs with a nil one"
- (sync-app)
- (is (nil? (t2/select-one-fn :description :model/DataApp :name "a"))))
- (testing "adding one counts as a change and is materialized"
- (is (=? {:changed 1} (sync-app :description "What this app does")))
- (is (= "What this app does" (t2/select-one-fn :description :model/DataApp :name "a"))))
- (testing "re-syncing the same description is not a change"
- (is (=? {:changed 0} (sync-app :description "What this app does"))))
- (testing "dropping it from the config clears the column"
- (is (=? {:changed 1} (sync-app)))
- (is (nil? (t2/select-one-fn :description :model/DataApp :name "a")))))))
-
-(deftest version-is-optional-and-tracked-like-other-metadata-test
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [sync-app (fn [& {:as app}]
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "a" (merge {:name "A" :path "index.js" :bundle "V1"} app)))))]
- (testing "an app that declares no version syncs as version 1"
- (sync-app)
- (is (= 1 (t2/select-one-fn :version :model/DataApp :name "a"))))
- (testing "declaring one counts as a change and is materialized"
- (is (=? {:changed 1} (sync-app :version 2)))
- (is (= 2 (t2/select-one-fn :version :model/DataApp :name "a"))))
- (testing "re-syncing the same version is not a change"
- (is (=? {:changed 0} (sync-app :version 2))))
- (testing "dropping it from the config falls back to version 1"
- (is (=? {:changed 1} (sync-app)))
- (is (= 1 (t2/select-one-fn :version :model/DataApp :name "a"))))
- (testing "an invalid version is a config error that keeps the app's last good state"
- (is (re-find #"\"version\" must be a positive whole number"
- (first (:config-errors (sync-app :version "1.0.0")))))
- (is (= 1 (t2/select-one-fn :version :model/DataApp :name "a")))))))
-
-(deftest metadata-edits-count-while-an-app-keeps-failing-test
- (testing "an app whose bundle is missing still stores metadata edits, so they count as changes"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [sync-app (fn [& {:as app}]
- (let [files (app-files "a" (merge {:name "A" :path "index.js" :bundle "unused"} app))]
- (data-app.sync/import-from-snapshot!
- (snapshot (dissoc files "data_apps/a/index.js")))))]
- (is (=? {:changed 1} (sync-app :description "First")) "the first failure is a change")
- (is (some? (t2/select-one-fn :sync_error :model/DataApp :name "a")))
- (testing "re-syncing the same failing app unchanged is not a change"
- (is (=? {:changed 0} (sync-app :description "First"))))
- (testing "editing the description is a change even though the bundle still fails"
- (is (=? {:changed 1} (sync-app :description "Second")))
- (is (= "Second" (t2/select-one-fn :description :model/DataApp :name "a"))
- "the edit is stored, which is why the pull cannot report it as a no-op"))))))
-
-(deftest switching-repos-prunes-old-apps-overrides-shared-adds-new-test
- (testing "syncing a different repo: drop apps only the old repo had, override shared slugs, add new ones"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- ;; Repo A: Foo + Bar
- (data-app.sync/import-from-snapshot!
- (snapshot (merge (app-files "foo" {:name "Foo" :path "index.js" :bundle "FOO"})
- (app-files "bar" {:name "Bar A" :path "index.js" :bundle "BAR-A"}))))
- (is (= #{"foo" "bar"} (t2/select-fn-set :name :model/DataApp)))
- ;; Repo B (Bar + Baz): the repo is the source of truth, so Foo (absent from B)
- ;; is pruned, Bar is overridden by slug, and Baz is added.
- (is (=? {:synced 2 :removed 1}
- (data-app.sync/import-from-snapshot!
- (snapshot (merge (app-files "bar" {:name "Bar B" :path "index.js" :bundle "BAR-B"})
- (app-files "baz" {:name "Baz" :path "index.js" :bundle "BAZ"}))))))
- (is (= #{"bar" "baz"} (t2/select-fn-set :name :model/DataApp))
- "Foo (only in repo A) is dropped; Baz (from repo B) is added")
- (let [bar (t2/select-one :model/DataApp :name "bar")]
- (is (= "Bar B" (:display_name bar))
- "Bar is overridden in place by repo B (shared slug)")
- (is (= "BAR-B" (String. ^bytes (:bundle bar) "UTF-8"))
- "Bar's cached bundle is repo B's")))))
-
-(deftest an-empty-repo-prunes-all-apps-test
- (testing "syncing a repo with no data_apps/ removes every app (the repo has none)"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "solo" {:name "Solo" :path "index.js" :bundle "S"})))
- (is (= #{"solo"} (t2/select-fn-set :name :model/DataApp)))
- (is (=? {:synced 0 :removed 1}
- (data-app.sync/import-from-snapshot! (snapshot {}))))
- (is (empty? (t2/select-fn-set :name :model/DataApp))))))
-
-(deftest remote-sync-preserves-unpublished-data-app-drafts-test
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (data-app.sync/ensure-draft! "draft-app")
- (is (=? {:removed 0}
- (data-app.sync/import-from-snapshot! (snapshot {}))))
- (is (true? (t2/select-one-fn :draft :model/DataApp :name "draft-app")))
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "draft-app" {:name "Draft" :path "index.js" :bundle "BUNDLE"})))
- (is (false? (t2/select-one-fn :draft :model/DataApp :name "draft-app")))
- (is (=? {:removed 1}
- (data-app.sync/import-from-snapshot! (snapshot {}))))
- (is (not (t2/exists? :model/DataApp :name "draft-app")))))
-
-(deftest repository-claims-a-draft-before-a-successful-import-test
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (data-app.sync/ensure-draft! "draft-app")
- (data-app.sync/import-from-snapshot!
- (snapshot {"data_apps/draft-app/data_app.yaml" "name: Draft\n"}))
- (is (false? (t2/select-one-fn :draft :model/DataApp :name "draft-app")))
- (is (=? {:removed 1}
- (data-app.sync/import-from-snapshot! (snapshot {}))))
- (is (not (t2/exists? :model/DataApp :name "draft-app")))))
-
-(deftest concurrent-draft-creation-is-safe-test
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [drafts (doall (repeatedly 2 #(future (data-app.sync/ensure-draft! "draft-app"))))]
- (doseq [draft drafts]
- @draft)
- (is (= 1 (t2/count :model/DataApp :name "draft-app"))))))
-
-(deftest remote-sync-prunes-never-successful-repository-apps-test
- (mt/with-model-cleanup [:model/DataApp]
- (data-app.sync/import-from-snapshot!
- (snapshot {"data_apps/broken/data_app.yaml" "name: Broken\npath: missing.js\n"}))
- (is (false? (t2/select-one-fn :draft :model/DataApp :name "broken")))
- (is (=? {:removed 1}
- (data-app.sync/import-from-snapshot! (snapshot {}))))
- (is (not (t2/exists? :model/DataApp :name "broken")))))
-
-(deftest a-broken-config-does-not-prune-the-existing-app-test
- (testing "a directory that still exists but whose data_app.yaml is now broken keeps the app (as a sync_error), it is not pruned"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "app" {:name "App" :path "index.js" :bundle "GOOD"})))
- (is (= "GOOD" (String. ^bytes (:bundle (t2/select-one :model/DataApp :name "app")) "UTF-8")))
- ;; Same directory, but its config no longer parses. The dir is still present,
- ;; so the app is kept — not treated as a removal — and its cached bundle stays.
- (let [result (data-app.sync/import-from-snapshot!
- (snapshot {"data_apps/app/data_app.yaml" "name: App\n" ; missing required "path"
- "data_apps/app/index.js" "GOOD"}))]
- (is (=? {:removed 0} result) "the app is not pruned")
- (is (= 1 (count (:config-errors result))))
- (is (=? {:changed 1} result)
- "marking the app failed counts as a change, so the pull isn't reported as a no-op"))
- (let [app (t2/select-one :model/DataApp :name "app")]
- (is (some? app) "the app survives a transiently broken config")
- (is (= "GOOD" (String. ^bytes (:bundle app) "UTF-8"))
- "its last-good cached bundle is retained")
- (is (str/includes? (:sync_error app) "path")
- "the parse failure is recorded on the row, so the UI shows it as failed rather than freshly synced")))))
-
-(deftest a-broken-config-for-a-brand-new-app-materializes-nothing-test
- (testing "an app whose config never parsed has no row to mark — it simply isn't materialized"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [result (data-app.sync/import-from-snapshot!
- (snapshot {"data_apps/newbie/data_app.yaml" "name: Newbie\n" ; missing required "path"
- "data_apps/newbie/index.js" "X"}))]
- (is (= 1 (count (:config-errors result))))
- (is (=? {:synced 0 :changed 0 :removed 0} result))
- (is (empty? (t2/select-fn-set :name :model/DataApp)))))))
-
-(defn- oversized-bundle ^String []
- (.repeat "a" (int (inc data-app.sync/max-bundle-bytes))))
-
-(deftest oversized-bundle-is-rejected-test
- (testing "a bundle over the size cap is rejected with a sync_error, no bundle cached"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "big" {:name "Big" :path "index.js"
- :bundle (oversized-bundle)})))
- (let [app (t2/select-one :model/DataApp :name "big")]
- (is (some? app) "the app still appears in the list")
- (is (nil? (:bundle app)) "no oversized bundle was cached")
- (is (str/includes? (:sync_error app) "MiB"))))))
-
-(deftest oversized-resync-keeps-the-previous-bundle-test
- (testing "an oversized re-sync sets sync_error but keeps the last good bundle"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "app" {:name "App" :path "index.js" :bundle "GOOD"})))
- (data-app.sync/import-from-snapshot!
- (snapshot (app-files "app" {:name "App" :path "index.js"
- :bundle (oversized-bundle)})))
- (let [app (t2/select-one :model/DataApp :name "app")]
- (is (= "GOOD" (String. ^bytes (:bundle app) "UTF-8"))
- "the previously cached bundle is retained")
- (is (str/includes? (:sync_error app) "MiB"))))))
-
-(deftest a-directory-without-a-config-is-not-an-app-test
- (testing "a data_apps/<dir> that ships a bundle but no data_app.yaml is not discovered — no app, no error"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [result (data-app.sync/import-from-snapshot!
- (snapshot {"data_apps/orphan/index.js" "BUNDLE"}))]
- (is (=? {:synced 0 :changed 0 :config-errors []} result))
- (is (empty? (t2/select-fn-set :name :model/DataApp)))))))
-
-(deftest an-unreadable-config-is-a-config-error-test
- (testing "a data_app.yaml the snapshot lists but can't read is isolated as a config-error, not a crash"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- ;; the config is listed in the tree, but reading its blob yields nothing
- (let [result (data-app.sync/import-from-snapshot!
- (snapshot {"data_apps/ghost/data_app.yaml" nil}))]
- (is (= 1 (count (:config-errors result))))
- (is (str/includes? (first (:config-errors result)) "data_apps/ghost/data_app.yaml"))
- (is (empty? (t2/select-fn-set :name :model/DataApp)))))))
-
-(deftest resource-failure-rolls-back-only-that-app-test
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [files (merge (app-files "broken" {:name "Broken" :path "index.js" :bundle "OLD"})
- (app-files "working" {:name "Working" :path "index.js" :bundle "OLD"}))
- ensure-resources! (mt/original-fn #'data-app.resources/ensure-resources!)]
- (data-app.sync/import-from-snapshot! (snapshot files))
- (let [collection-id (t2/select-one-fn :resource_collection_id :model/DataApp :name "broken")
- old-name (t2/select-one-fn :name :model/Collection :id collection-id)]
- ; inject an error into `ensure-resources!` for the "broken" app
- (mt/with-dynamic-fn-redefs [data-app.resources/ensure-resources!
- (fn [app]
- (ensure-resources! app)
- (when (= "broken" (:name app))
- (t2/update! :model/Collection :id collection-id {:name "Partial update"})
- (throw (ex-info "Resource provisioning failed." {}))))]
- (data-app.sync/import-from-snapshot!
- (snapshot (assoc files "data_apps/broken/index.js" "NEW" "data_apps/working/index.js" "NEW"))))
- ; the "broken" app bundle with sync error should be rolled back to "OLD"
- (is (= "OLD" (String. ^bytes (t2/select-one-fn :bundle :model/DataApp :name "broken") "UTF-8")))
- ; the collection name should be rolled back to the old name
- (is (= old-name (t2/select-one-fn :name :model/Collection :id collection-id)))
- ; the thrown exception in ensure-resources! should be reported in the sync error
- (is (= "Resource provisioning failed." (t2/select-one-fn :sync_error :model/DataApp :name "broken")))
- ; the "working" app without sync errors should not be rolled back to "OLD"
- (is (= "NEW" (String. ^bytes (t2/select-one-fn :bundle :model/DataApp :name "working") "UTF-8")))))))
-
-(deftest pruning-failure-does-not-roll-back-synced-apps-test
- (mt/with-premium-features #{:data-apps}
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [working-files (app-files "working" {:name "Working" :path "index.js" :bundle "OLD"})
- files (merge working-files (app-files "removed" {:name "Removed" :path "index.js" :bundle "OLD"}))
- prune! (mt/original-fn #'data-apps.db/delete-data-apps-not-named!)]
- (data-app.sync/import-from-snapshot! (snapshot files))
- ; inject an error into deleting data apps
- (mt/with-dynamic-fn-redefs [data-apps.db/delete-data-apps-not-named!
- (fn [slugs]
- (prune! slugs)
- (throw (ex-info "Pruning failed." {})))]
- ; sync should report the error from pruning step
- (is (=? {:synced 1 :changed 1 :removed 0 :pruning-error "Pruning failed."}
- (data-app.sync/sync-from-snapshot!
- (snapshot (assoc working-files "data_apps/working/index.js" "NEW"))))))
- ; a pruning failure should not roll back the updated app's bundle
- (is (= "NEW" (String. ^bytes (t2/select-one-fn :bundle :model/DataApp :name "working") "UTF-8")))
- (is (t2/exists? :model/DataApp :name "removed"))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.clj
deleted file mode 100644
index f882240b4cc..00000000000
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/data_app_pull_test.clj
+++ /dev/null
@@ -1,126 +0,0 @@
-(ns metabase-enterprise.remote-sync.data-app-pull-test
- "A remote-sync pull's outcome (the \"N changes\" summary the admin sees) must
- count data-app changes. Data apps live under `data_apps/` — outside serdes — and
- are materialized separately (`metabase-enterprise.data-apps.sync`), so without
- `impl/fold-data-app-changes` a pull whose only changes are data apps reports
- `pull-skipped` / `count 0`. These cover the three pull shapes: data-app-only,
- mixed (serdes + data apps), and serdes-only."
- (:require
- [clojure.test :refer :all]
- [metabase-enterprise.data-apps.sync :as data-apps.sync]
- [metabase-enterprise.remote-sync.impl :as impl]
- [metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]
- [metabase-enterprise.remote-sync.source.protocol :as source.p]
- [metabase-enterprise.remote-sync.test-helpers :as rs.test]
- [metabase.search.test-util :as search.tu]
- [metabase.test :as mt]
- [metabase.test.fixtures :as fixtures]
- [toucan2.core :as t2]))
-
-(set! *warn-on-reflection* true)
-
-(use-fixtures :once (fixtures/initialize :db))
-(use-fixtures :each rs.test/clean-remote-sync-state rs.test/commit-with-temp)
-(defn- import-at!
- "Run `import!` against the source's snapshot at `version`, complete the task (so
- `last-version` advances for the next pull), and return the result."
- [src version & {:keys [force?] :or {force? false}}]
- (let [task (t2/insert-returning-pk! :model/RemoteSyncTask
- {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- result (impl/import! (source.p/snapshot-at src version) task :force? force?)]
- (impl/handle-task-result! result task)
- result))
-
-(defn- app-tree!
- "Repo files for one data app: its `data_app.yaml` + a bundle at `dist/index.js`."
- [slug bundle]
- {(str "data_apps/" slug "/data_app.yaml") (str "name: " slug "\npath: dist/index.js\n")
- (str "data_apps/" slug "/dist/index.js") bundle})
-
-;; One self-contained serdes entity (a bare collection — no DB deps), used as the
-;; "data" change. Reuses the path/content shape the mock harness imports cleanly.
-(def ^:private coll-path
- "collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/M-Q4pcV0qkiyJ0kiSWECl_some_collection.yaml")
-(def ^:private coll-file
- {coll-path (rs.test/generate-collection-yaml "M-Q4pcV0qkiyJ0kiSWECl" "Some Collection")})
-
-(defn- pull-outcome!
- "Establish `v0` as the baseline, then pull `v1` (a normal, non-forced pull) and
- return its `:outcome`."
- [v0 v1]
- (let [src (rs.test/versioned-source :trees {"v0" v0 "v1" v1} :current "v0")]
- (is (= :success (:status (import-at! src "v0" :force? true))) "baseline import succeeds")
- (let [result (import-at! src "v1")]
- (is (= :success (:status result)) "pull succeeds")
- (:outcome result))))
-
-(deftest data-app-only-pull-counts-app-changes-test
- (testing "a pull whose only change is a data app is reported as a real pull, not pull-skipped / 0 changes"
- (search.tu/with-index-disabled
- (mt/with-premium-features #{:data-apps}
- (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [outcome (pull-outcome! (app-tree! "sales" "BUNDLE-V1")
- (app-tree! "sales" "BUNDLE-V2"))]
- (is (= "pulled" (:kind outcome)) "not reported as skipped")
- (is (= 1 (:count outcome)) "the one changed data app is counted"))))))))
-
-(deftest data-only-pull-excludes-unchanged-apps-test
- (testing "a serdes-only pull counts serdes content; unchanged data apps add nothing"
- (search.tu/with-index-disabled
- (mt/with-premium-features #{:data-apps}
- (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- ;; v1 adds a collection; the data app is byte-for-byte the same as v0.
- (let [app (app-tree! "sales" "BUNDLE")
- outcome (pull-outcome! app (merge coll-file app))]
- (is (= "pulled" (:kind outcome)))
- (is (= 1 (:count outcome)) "the collection counts; the unchanged app adds 0"))))))))
-
-(deftest mixed-pull-counts-serdes-and-apps-test
- (testing "a mixed pull counts serdes content AND the changed data apps"
- (search.tu/with-index-disabled
- (mt/with-premium-features #{:data-apps}
- (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- ;; v1 adds a collection AND changes the data app's bundle.
- (let [outcome (pull-outcome! (app-tree! "ops" "BUNDLE")
- (merge coll-file (app-tree! "ops" "BUNDLE-V2")))]
- (is (= "pulled" (:kind outcome)))
- (is (= 2 (:count outcome)) "the collection (1) plus the changed data app (1)"))))))))
-
-(deftest data-apps-land-with-the-version-test
- (testing "a pull records the data apps at the snapshot version it commits as the sync base"
- (search.tu/with-index-disabled
- (mt/with-premium-features #{:data-apps}
- (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [src (rs.test/versioned-source :trees {"v0" (app-tree! "sales" "BUNDLE-V1")
- "v1" (app-tree! "sales" "BUNDLE-V2")}
- :current "v0")]
- (is (= :success (:status (import-at! src "v0" :force? true))))
- (is (= :success (:status (import-at! src "v1"))))
- (is (= "v1" (remote-sync.task/last-version)))
- (is (= "v1" (t2/select-one-fn :last_synced_sha :model/DataApp :name "sales"))))))))))
-
-(deftest data-app-failure-rolls-back-the-version-test
- (testing "a crash while materializing data apps rolls the version back, so the next pull redoes the import
- instead of skipping a version whose data apps never landed"
- (search.tu/with-index-disabled
- (mt/with-premium-features #{:data-apps}
- (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [src (rs.test/versioned-source :trees {"v0" (app-tree! "sales" "BUNDLE-V1")
- "v1" (app-tree! "sales" "BUNDLE-V2")}
- :current "v0")]
- (is (= :success (:status (import-at! src "v0" :force? true))))
- (let [v0-hash (t2/select-one-fn :bundle_hash :model/DataApp :name "sales")]
- (mt/with-dynamic-fn-redefs [data-apps.sync/sync-from-snapshot! (fn [_] (throw (ex-info "materialization died" {})))]
- (is (= :error (:status (import-at! src "v1")))))
- (is (= "v0" (remote-sync.task/last-version)) "the failed pull is not the sync base")
- (is (= v0-hash (t2/select-one-fn :bundle_hash :model/DataApp :name "sales"))
- "the v0 app is untouched"))
- (is (=? {:status :success :outcome {:kind "pulled"}} (import-at! src "v1"))
- "the next pull redoes the import rather than skipping v1")
- (is (= "v1" (remote-sync.task/last-version)))
- (is (= "v1" (t2/select-one-fn :last_synced_sha :model/DataApp :name "sales"))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj
index f223099455a..42e2fd494e8 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj
@@ -1,77 +1,154 @@
(ns metabase-enterprise.remote-sync.data-apps-test
- "Integration coverage for the data-apps hook in the remote-sync import pipeline:
- importing a repo materializes data apps from its `data_apps/` directory."
+ "Data apps are serdes entities: remote sync imports and exports them like any other, with each app's bundle as a
+ resource file next to its `data_app.yaml`."
(:require
[clojure.test :refer :all]
- [metabase-enterprise.data-apps.models.data-app]
[metabase-enterprise.remote-sync.impl :as impl]
+ [metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.remote-sync.test-helpers :as test-helpers]
+ [metabase.search.test-util :as search.tu]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
+ [metabase.util.yaml :as yaml]
[toucan2.core :as t2]))
+(set! *warn-on-reflection* true)
+
(use-fixtures :once (fixtures/initialize :db))
-(use-fixtures :each (fn [f] (test-helpers/clean-remote-sync-state f)))
-(comment metabase-enterprise.data-apps.models.data-app/keep-me)
+(use-fixtures :each test-helpers/clean-remote-sync-state test-helpers/commit-with-temp)
+
+(def ^:private sales-eid "w8YkMQBwU4kOD0-pZ1nJk")
+
+(defn- app-tree
+ "Repo files for the data app `slug` with entity id `eid`: its `data_app.yaml` and its bundle at `dist/index.js`."
+ [eid slug bundle]
+ {(str "data_apps/" slug "/data_app.yaml")
+ (yaml/generate-string {:serdes/meta [{:model "DataApp" :id eid :label slug}]
+ :entity_id eid
+ :slug slug
+ :name "Sales"
+ :path "dist/index.js"})
+ (str "data_apps/" slug "/dist/index.js") bundle})
+
+(defn- new-task! [sync-task-type]
+ (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type sync-task-type :initiated_by (mt/user->id :rasta)}))
+
+(defn- import-at!
+ "Run `import!` against the source's snapshot at `version`, complete the task (so `last-version` advances for the
+ next pull), and return the result."
+ [src version & {:keys [force?] :or {force? false}}]
+ (let [task (new-task! "import")
+ result (impl/import! (source.p/snapshot-at src version) task :force? force?)]
+ (impl/handle-task-result! result task)
+ result))
+
+(defn- export!
+ "Export to `mock` under a fresh task, complete it, and return the result."
+ [mock]
+ (t2/delete! :model/RemoteSyncTask)
+ (let [task (new-task! "export")
+ result (impl/export! (source.p/snapshot mock) task "export")]
+ (impl/handle-task-result! result task)
+ result))
+
+(defn- bundle-text [slug]
+ (when-let [^bytes bundle (t2/select-one-fn :bundle [:model/DataApp :bundle] :name slug)]
+ (String. bundle "UTF-8")))
+
+(defmacro ^:private with-data-apps-sync [& body]
+ `(search.tu/with-index-disabled
+ (mt/with-premium-features #{:data-apps}
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true remote-sync-transforms false]
+ (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
+ ~@body)))))
+
+(deftest pull-imports-and-prunes-data-apps-test
+ (with-data-apps-sync
+ (let [src (test-helpers/versioned-source :trees {"v0" (app-tree sales-eid "sales" "BUNDLE-V1")
+ "v1" {"README.md" "x"}}
+ :current "v0")]
+ (testing "a pull imports the app and its bundle"
+ (is (=? {:status :success :outcome {:kind "pulled" :count 1}} (import-at! src "v0" :force? true)))
+ (is (=? {:entity_id sales-eid :display_name "Sales" :draft false :resource_collection_id pos-int?}
+ (t2/select-one :model/DataApp :name "sales")))
+ (is (= "BUNDLE-V1" (bundle-text "sales"))))
+ (testing "a pull whose repo no longer has the app removes it"
+ (is (= :success (:status (import-at! src "v1"))))
+ (is (not (t2/exists? :model/DataApp :name "sales")))))))
-(defn- import! [files]
- ;; the import's data-app materialization is gated on :data-apps; enable it so apps sync.
- (mt/with-premium-features #{:data-apps}
- (let [source (test-helpers/create-mock-source :initial-files files)
- task-id (t2/insert-returning-pk! :model/RemoteSyncTask
- {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- ;; force: the mock source reports a constant version, so without this the
- ;; second import in a test would hit the "version unchanged" skip branch
- result (impl/import! (source.p/snapshot source) task-id :force? true)]
- ;; calling import! directly leaves the task "running"; mark it ended so a
- ;; subsequent import in the same test isn't blocked by the running-task guard
- (t2/update! :model/RemoteSyncTask task-id {:ended_at :%now})
- result)))
+(deftest bundle-only-pull-updates-the-bundle-test
+ (testing "a pull that changes only an app's bundle file falls back to a full import, so the new bundle lands"
+ (with-data-apps-sync
+ (let [src (test-helpers/versioned-source :trees {"v0" (app-tree sales-eid "sales" "BUNDLE-V1")
+ "v1" (app-tree sales-eid "sales" "BUNDLE-V2")}
+ :current "v0")]
+ (is (= :success (:status (import-at! src "v0" :force? true))))
+ (is (=? {:status :success :outcome {:kind "pulled"}} (import-at! src "v1")))
+ (is (= "v1" (remote-sync.task/last-version)))
+ (is (= "BUNDLE-V2" (bundle-text "sales")))))))
-(deftest import-materializes-data-apps-test
- (testing "a remote-sync import materializes data apps from data_apps/ alongside serdes content"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (let [files {"main" {"collections/c/c.yaml"
- (test-helpers/generate-collection-yaml "data-apps-test-collx" "DA Coll")
- "data_apps/sales/data_app.yaml" "name: Sales\npath: ./dist/index.js\n"
- "data_apps/sales/dist/index.js" "SALESBUNDLE"}}
- result (import! files)]
- (is (= :success (:status result)))
- (let [app (t2/select-one :model/DataApp :name "sales")]
- (is (some? app) "the data app from the repo was materialized by the import")
- (is (= "Sales" (:display_name app)))
- (is (= "data_apps/sales/dist/index.js" (:bundle_path app)))
- (is (true? (:enabled app)))
- (is (= "SALESBUNDLE" (String. ^bytes (:bundle app) "UTF-8"))))))))
+(deftest export-writes-and-removes-data-app-files-test
+ (with-data-apps-sync
+ (let [mock (test-helpers/create-mock-source :initial-files {"main" {}})
+ repo #(get @(:files-atom mock) "main")]
+ (mt/user-http-request :crowberto :post 200 "apps" {:name "sales"
+ :display_name "Sales"
+ :bundle_path "dist/index.js"
+ :bundle "BUNDLE"})
+ (testing "a created app is exported as its manifest and its bundle file"
+ (is (= :success (:status (export! mock))))
+ (is (=? {:slug "sales" :name "Sales" :path "dist/index.js"}
+ (yaml/parse-string (get (repo) "data_apps/sales/data_app.yaml"))))
+ (is (= "BUNDLE" (get (repo) "data_apps/sales/dist/index.js"))))
+ (testing "an updated bundle is exported"
+ (mt/user-http-request :crowberto :put 200 "apps/sales" {:bundle "BUNDLE-V2"})
+ (is (= :success (:status (export! mock))))
+ (is (= "BUNDLE-V2" (get (repo) "data_apps/sales/dist/index.js"))))
+ (testing "a deleted app's files are removed from the repo, and the other apps' files stay"
+ (mt/user-http-request :crowberto :post 200 "apps" {:name "ops"
+ :display_name "Ops"
+ :bundle_path "app.js"
+ :bundle "OPS"})
+ (mt/user-http-request :crowberto :delete 204 "apps/sales")
+ (is (= :success (:status (export! mock))))
+ (is (= #{"data_apps/ops/data_app.yaml" "data_apps/ops/app.js"}
+ (into #{} (filter #(re-find #"^data_apps/" %)) (keys (repo)))))))))
-(deftest import-prunes-data-apps-absent-from-repo-test
- (testing "an import whose repo no longer has an app dir prunes that app (the repo is the source of truth)"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- (import! {"main" {"data_apps/gone/data_app.yaml" "name: Gone\npath: ./i.js\n"
- "data_apps/gone/i.js" "X"
- "data_apps/kept/data_app.yaml" "name: Kept\npath: ./i.js\n"
- "data_apps/kept/i.js" "K"}})
- (is (= #{"gone" "kept"} (t2/select-fn-set :name :model/DataApp)))
- (import! {"main" {"data_apps/kept/data_app.yaml" "name: Kept\npath: ./i.js\n"
- "data_apps/kept/i.js" "K"}})
- (is (nil? (t2/select-one :model/DataApp :name "gone"))
- "the app absent from the later import is pruned")
- (is (some? (t2/select-one :model/DataApp :name "kept"))
- "the still-present app is kept"))))
+(deftest export-keeps-files-serialization-does-not-own-test
+ (testing "an export rewrites an app's manifest and bundle but leaves the app's source next to them alone"
+ (with-data-apps-sync
+ (let [source {"data_apps/sales/src/App.tsx" "export const App = () => null;"
+ "data_apps/sales/package.json" "{}"
+ "data_apps/sales/pnpm-lock.yaml" "lockfileVersion: '9.0'\n"
+ "data_apps/sales/deploy/k8s.yaml" "a: 1\n---\nb: [unterminated\n"}
+ mock (test-helpers/create-mock-source
+ :initial-files {"main" (merge (app-tree sales-eid "sales" "BUNDLE-V1") source)})
+ repo #(get @(:files-atom mock) "main")]
+ (is (= :success (:status (import-at! mock "main" :force? true))))
+ (mt/user-http-request :crowberto :put 200 "apps/sales" {:bundle "BUNDLE-V2"})
+ (mt/user-http-request :crowberto :post 200 "apps" {:name "ops"
+ :display_name "Ops"
+ :bundle_path "app.js"
+ :bundle "OPS"})
+ (is (= :success (:status (export! mock))))
+ (is (= "BUNDLE-V2" (get (repo) "data_apps/sales/dist/index.js")))
+ (is (= source (select-keys (repo) (keys source))))
+ (testing "deleting the app removes only the files serialization owns"
+ (mt/user-http-request :crowberto :delete 204 "apps/sales")
+ (is (= :success (:status (export! mock))))
+ (is (= (into #{"data_apps/ops/data_app.yaml" "data_apps/ops/app.js"} (keys source))
+ (into #{} (filter #(re-find #"^data_apps/" %)) (keys (repo))))))))))
-(deftest deletion-only-pull-counts-the-removal-test
- (testing "a pull whose only change is removing an app still reports as a pull — the removal is counted, not a no-op"
- (mt/with-model-cleanup [:model/DataApp :model/Collection :model/PermissionsGroup]
- ;; No serdes content, so the outcome's count comes purely from data apps.
- (import! {"main" {"README.md" "x"
- "data_apps/gone/data_app.yaml" "name: Gone\npath: ./i.js\n"
- "data_apps/gone/i.js" "X"}})
- (is (= #{"gone"} (t2/select-fn-set :name :model/DataApp)))
- ;; Same repo minus the app dir: nothing is upserted (`:changed` 0), one app removed.
- (let [result (import! {"main" {"README.md" "x"}})]
- (is (= :success (:status result)))
- (is (empty? (t2/select-fn-set :name :model/DataApp))
- "the app is pruned")
- (is (pos? (:count (:outcome result)))
- "the removal is folded into the pull outcome, so it isn't reported as pull-skipped/0")))))
+(deftest pull-refuses-to-delete-an-unpushed-app-test
+ (testing "a pull whose repo lacks an app created here but not pushed yet is a conflict, not a silent delete"
+ (with-data-apps-sync
+ (let [app (app-tree sales-eid "sales" "BUNDLE")
+ src (test-helpers/versioned-source :trees {"v0" app "v1" (assoc app "README.md" "x")} :current "v0")]
+ (is (= :success (:status (import-at! src "v0" :force? true))))
+ (mt/user-http-request :crowberto :post 200 "apps" {:name "ops"
+ :display_name "Ops"
+ :bundle_path "app.js"
+ :bundle "OPS"})
+ (is (=? {:status :conflict} (import-at! src "v1")))
+ (is (t2/exists? :model/DataApp :name "ops"))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
index 249dc172836..5afdf0d3589 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
@@ -44,7 +44,7 @@
(deftest all-specs-have-valid-eligibility-test
(testing "Every spec has a valid eligibility type"
- (let [valid-eligibility-types #{:collection :published-table :parent-table :parent :setting :library-synced}]
+ (let [valid-eligibility-types #{:collection :published-table :parent-table :parent :setting :library-synced :always}]
(doseq [[model-key spec] spec/remote-sync-specs]
(testing (str "Spec for " model-key)
(is (contains? valid-eligibility-types (get-in spec [:eligibility :type]))
@@ -137,7 +137,8 @@
(is (contains? types "TransformTest"))
(is (contains? types "Glossary"))
(is (contains? types "Action"))
- (is (= 16 (count types))))))
+ (is (contains? types "DataApp"))
+ (is (= 17 (count types))))))
(deftest specs-by-identity-type-test
(testing "specs-by-identity-type filters correctly"
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 2b49d5e3fc4..bf9f55be513 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -267,7 +267,7 @@ width: fixed
:or {branch "main"
fail-mode nil
initial-files nil
- managed-dirs ingest/legal-top-level-paths}}]
+ managed-dirs ingest/replaced-top-level-paths}}]
(let [default-files {"main" {"collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/M-Q4pcV0qkiyJ0kiSWECl_some_collection.yaml"
(generate-collection-yaml "M-Q4pcV0qkiyJ0kiSWECl" "Some Collection")
@@ -297,7 +297,7 @@ width: fixed
model a base orphaned by a force-push/rebase). Committing (via `open-commit`) records the written set
under a fresh version, advances `:current`, and returns the new version so an export can fast-forward onto it."
[& {:keys [current trees branch managed-dirs]
- :or {current "v-remote" branch "main" managed-dirs ingest/legal-top-level-paths}}]
+ :or {current "v-remote" branch "main" managed-dirs ingest/replaced-top-level-paths}}]
(let [managed (set managed-dirs)
state (atom {:current current :trees (or trees {}) :counter 0})
diff-trees (fn [old-tree new-tree]
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
index 76f1959b6d3..5131b320585 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
@@ -54,7 +54,6 @@
:model/CollectionBookmark
:model/ContentTranslation
:model/DashboardBookmark
- :model/DataApp
:model/DataComplexityScore
:model/DataPermissions
:model/DatabaseRouter
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
index 74a11b1e898..695147eecd6 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
@@ -148,10 +148,11 @@
fixture in this shared baseline. OsiAiContext is covered (in-memory + on-disk) by
metabase-enterprise.serialization.v2.osi-ai-context-test. TransformTest is covered by
transform-test-round-trip-test and transform-test-expectations-round-trip-test in
- metabase-enterprise.serialization.v2.e2e-test, and stays out of the baseline until
- @metabase/representations publishes a schema for it — the baseline is validated against that
+ metabase-enterprise.serialization.v2.e2e-test, and DataApp by
+ metabase-enterprise.data-apps.serialization-test; both stay out of the baseline until
+ @metabase/representations publishes a schema for them — the baseline is validated against that
package, which refuses a model it does not know."
- #{"OsiAiContext" "TransformTest"})
+ #{"DataApp" "OsiAiContext" "TransformTest"})
(defn add-to-baseline!
"Use this within v2.extract-test where relevant to add their fixtures to the baseline."
diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.ts
index f2348484c48..03c6d6c0fc5 100644
--- a/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.ts
+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.ts
@@ -4,6 +4,7 @@ import path from "node:path";
import { load as parseYaml } from "js-yaml";
export type DataAppManifest = {
+ slug?: string;
allowed_hosts?: string[];
};
@@ -34,6 +35,21 @@ const parseAllowedHosts = (
return value;
};
+const parseSlug = (
+ value: unknown,
+ manifestPath: string,
+): string | undefined => {
+ if (value == null) {
+ return undefined;
+ }
+
+ if (!isString(value)) {
+ throw new Error(`${manifestPath}: "slug" must be a string.`);
+ }
+
+ return value;
+};
+
export const readManifest = (
appRoot: string,
): { manifestPath: string; manifest: DataAppManifest } | null => {
@@ -57,12 +73,13 @@ export const readManifest = (
);
}
- const raw: { allowed_hosts?: unknown } =
+ const raw: { slug?: unknown; allowed_hosts?: unknown } =
typeof parsed === "object" && parsed !== null ? parsed : {};
return {
manifestPath,
manifest: {
+ slug: parseSlug(raw.slug, manifestPath),
allowed_hosts: parseAllowedHosts(raw.allowed_hosts, manifestPath),
},
};
diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.unit.spec.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.unit.spec.ts
index a71b090e99d..beed6289a4c 100644
--- a/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.unit.spec.ts
+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/read-manifest.unit.spec.ts
@@ -44,6 +44,20 @@ describe("readManifest", () => {
expect(() => readManifest(APP_ROOT)).toThrow(/Could not parse/);
});
+ describe("slug", () => {
+ it("reads the slug", () => {
+ setup({ [YAML_PATH]: "slug: sales-ops\n" });
+
+ expect(readManifest(APP_ROOT)?.manifest.slug).toBe("sales-ops");
+ });
+
+ it("throws on a non-string slug", () => {
+ setup({ [YAML_PATH]: "slug: [a]\n" });
+
+ expect(() => readManifest(APP_ROOT)).toThrow(/"slug" must be a string/);
+ });
+ });
+
describe("allowed_hosts", () => {
it("reads allowed_hosts as a list of strings", () => {
setup({
diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/validate-manifest.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/validate-manifest.ts
index 0fc57d76cf2..9c4919d4091 100644
--- a/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/validate-manifest.ts
+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-dev/config/validate-manifest.ts
@@ -36,7 +36,7 @@ const compareAllowedHosts = (
};
/**
- * Not a full validator: the manifest rules live in the backend's `parse-app-config`
+ * Not a full validator: the manifest rules live in the backend's data-apps schemas
* and a copy here would drift. This only adds what the backend cannot see —
* this machine's disk, and what the dev server booted with.
*/
diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/sync.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/sync.ts
index 0f26582cfe3..fc65ac1c0e9 100644
--- a/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/sync.ts
+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/sync.ts
@@ -1,5 +1,7 @@
import path from "node:path";
+import { readManifest } from "../data-app-dev/config/read-manifest";
+
import { discoverActions, discoverQueries } from "./discover";
import { isPositiveInteger } from "./guards";
import {
@@ -161,7 +163,12 @@ export async function syncResources({
]);
const lockfile = readResourceLockfile(appRoot);
const client = new MetabaseClient(metabaseUrl, apiKey);
- const slug = path.basename(appRoot);
+ const slug = readManifest(appRoot)?.manifest.slug;
+ if (!slug) {
+ throw new Error(
+ `${path.join(appRoot, "data_app.yaml")} must declare the app's "slug".`,
+ );
+ }
const app = await client.ensureDraft(slug);
if (!isPositiveInteger(app.resource_collection_id)) {
throw new Error(`Data app ${slug} does not have a resource collection.`);
diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/setup.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/setup.ts
index 58fd7e8868a..a8919b6cee7 100644
--- a/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/setup.ts
+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/setup.ts
@@ -27,6 +27,10 @@ export function makeApp() {
fs.mkdirSync(path.join(appRoot, "queries"));
fs.mkdirSync(path.join(appRoot, "actions"));
+ fs.writeFileSync(
+ path.join(appRoot, "data_app.yaml"),
+ `slug: ${path.basename(appRoot)}\n`,
+ );
const packageRoot = path.join(
appRoot,
diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/sync.unit.spec.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/sync.unit.spec.ts
index 3cf88b7187a..f81667f096f 100644
--- a/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/sync.unit.spec.ts
+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-query-sync/tests/sync.unit.spec.ts
@@ -1,3 +1,4 @@
+import fs from "node:fs";
import path from "node:path";
import { discoverQueries } from "../discover";
@@ -14,6 +15,45 @@ import {
describe("query synchronization", () => {
setupResourceSyncTests();
+ it("synchronizes the app named by the manifest's slug, not its directory", async () => {
+ const appRoot = makeApp();
+ fs.writeFileSync(path.join(appRoot, "data_app.yaml"), "slug: other-app\n");
+ const fetchSpy = jest.spyOn(global, "fetch").mockResolvedValue(
+ new Response("Not found.", {
+ status: 404,
+ headers: { "content-type": "text/plain" },
+ }),
+ );
+
+ await expect(
+ syncResources({
+ appRoot,
+ metabaseUrl: "http://metabase.test",
+ apiKey: "secret",
+ log: jest.fn(),
+ }),
+ ).rejects.toThrow(/apps\/other-app\/draft/);
+ expect(String(fetchSpy.mock.calls[0][0])).toBe(
+ "http://metabase.test/api/apps/other-app/draft",
+ );
+ });
+
+ it("refuses an app whose manifest declares no slug", async () => {
+ const appRoot = makeApp();
+ fs.writeFileSync(path.join(appRoot, "data_app.yaml"), "name: App\n");
+ const fetchSpy = jest.spyOn(global, "fetch");
+
+ await expect(
+ syncResources({
+ appRoot,
+ metabaseUrl: "http://metabase.test",
+ apiKey: "secret",
+ log: jest.fn(),
+ }),
+ ).rejects.toThrow(/must declare the app's "slug"/);
+ expect(fetchSpy).not.toHaveBeenCalled();
+ });
+
it("identifies the request that failed", async () => {
const appRoot = makeApp();
const slug = path.basename(appRoot);
diff --git a/enterprise/frontend/src/metabase-enterprise/data_apps/admin/ManageDataAppsPage.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/data_apps/admin/ManageDataAppsPage.unit.spec.tsx
index 1d5b4cf9411..5b499d3bb34 100644
--- a/enterprise/frontend/src/metabase-enterprise/data_apps/admin/ManageDataAppsPage.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/data_apps/admin/ManageDataAppsPage.unit.spec.tsx
@@ -149,34 +149,16 @@ describe("ManageDataAppsPage", () => {
).toBeInTheDocument();
});
- it("renders each sync status: a synced sha, a failure with its reason, and never-synced", async () => {
+ it("labels drafts", async () => {
setup({
apps: [
- createMockDataApp({
- id: 1,
- display_name: "Synced App",
- last_synced_sha: "abcdef0",
- }),
- createMockDataApp({
- id: 2,
- display_name: "Failed App",
- sync_error: "boom: bad manifest",
- }),
- createMockDataApp({
- id: 3,
- display_name: "New App",
- last_synced_sha: null,
- sync_error: null,
- }),
+ createMockDataApp({ id: 1, display_name: "Live App" }),
+ createMockDataApp({ id: 2, display_name: "Draft App", draft: true }),
],
});
- expect(await screen.findByText("Synced abcdef0")).toBeInTheDocument();
- expect(screen.getByText("Sync failed")).toHaveAttribute(
- "title",
- "boom: bad manifest",
- );
- expect(screen.getByText("Not synced yet")).toBeInTheDocument();
+ expect(await screen.findByText("Draft App")).toBeInTheDocument();
+ expect(screen.getAllByText("Draft")).toHaveLength(1);
});
it("shows how many hosts an app is allowed to reach", async () => {
diff --git a/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.tsx b/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.tsx
index ce5ed024514..fd1c2a0bf26 100644
--- a/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.tsx
@@ -1,7 +1,6 @@
import { t } from "ttag";
import { Group, Stack, Text } from "metabase/ui";
-import type { MetabaseColorKey } from "metabase/ui/colors/types";
import * as Urls from "metabase/urls";
import type { DataApp } from "metabase-types/api";
@@ -18,37 +17,8 @@ const Bullet = () => (
</Text>
);
-function getSyncStatus({ sync_error, last_synced_sha }: DataApp): {
- label: string;
- color: MetabaseColorKey;
- title?: string;
-} {
- if (sync_error) {
- return { label: t`Sync failed`, color: "error", title: sync_error };
- }
-
- if (last_synced_sha) {
- return {
- label: t`Synced ${last_synced_sha.slice(0, 7)}`,
- color: "text-secondary",
- };
- }
-
- return { label: t`Not synced yet`, color: "text-tertiary" };
-}
-
-const SyncStatus = ({ app }: Props) => {
- const { label, color, title } = getSyncStatus(app);
-
- return (
- <Text size="sm" c={color} title={title} lh="1.4">
- {label}
- </Text>
- );
-};
-
export const DataAppSummary = ({ app }: Props) => {
- const isOpenable = app.enabled && !app.sync_error && !app.outdated;
+ const isOpenable = app.enabled && !app.draft && !app.outdated;
return (
<Group align="center" flex="1" wrap="nowrap" miw={0}>
@@ -92,9 +62,15 @@ export const DataAppSummary = ({ app }: Props) => {
{Urls.dataApp(app.name)}
</Text>
- <Bullet />
+ {app.draft && (
+ <>
+ <Bullet />
- <SyncStatus app={app} />
+ <Text size="sm" c="text-tertiary" lh="1.4">
+ {t`Draft`}
+ </Text>
+ </>
+ )}
{app.allowed_hosts.length > 0 && (
<>
diff --git a/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.unit.spec.tsx
index 3b502fd17da..8684bb145a4 100644
--- a/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/data_apps/admin/components/DataAppSummary/DataAppSummary.unit.spec.tsx
@@ -31,13 +31,13 @@ describe("DataAppSummary", () => {
).not.toBeInTheDocument();
});
- it("renders a sync-failed app's name as plain text (the link would lead to a dead end)", () => {
+ it("renders a draft's name as plain text (it has no bundle to open)", () => {
renderWithProviders(
<DataAppSummary
app={createMockDataApp({
display_name: "Sales",
enabled: true,
- sync_error: "Bundle file not found",
+ draft: true,
})}
/>,
);
@@ -93,54 +93,19 @@ describe("DataAppSummary", () => {
});
});
- describe("sync status", () => {
- it("shows the short synced SHA when the app has synced", () => {
+ describe("draft badge", () => {
+ it("labels a draft", () => {
renderWithProviders(
- <DataAppSummary
- app={createMockDataApp({
- last_synced_sha: "0123456789abcdef",
- sync_error: null,
- })}
- />,
- );
-
- expect(screen.getByText("Synced 0123456")).toBeInTheDocument();
- });
-
- it("shows a failure label with the error as a tooltip when the sync failed", () => {
- renderWithProviders(
- <DataAppSummary
- app={createMockDataApp({ sync_error: "boom: bad manifest" })}
- />,
- );
-
- const status = screen.getByText("Sync failed");
- expect(status).toBeInTheDocument();
- expect(status).toHaveAttribute("title", "boom: bad manifest");
- });
-
- it("prefers the failure label over the SHA when both are present", () => {
- renderWithProviders(
- <DataAppSummary
- app={createMockDataApp({
- last_synced_sha: "0123456789abcdef",
- sync_error: "boom",
- })}
- />,
+ <DataAppSummary app={createMockDataApp({ draft: true })} />,
);
- expect(screen.getByText("Sync failed")).toBeInTheDocument();
- expect(screen.queryByText(/^Synced/)).not.toBeInTheDocument();
+ expect(screen.getByText("Draft")).toBeInTheDocument();
});
- it("shows 'Not synced yet' when the app has never synced", () => {
- renderWithProviders(
- <DataAppSummary
- app={createMockDataApp({ last_synced_sha: null, sync_error: null })}
- />,
- );
+ it("doesn't label an app that isn't a draft", () => {
+ renderWithProviders(<DataAppSummary app={createMockDataApp()} />);
- expect(screen.getByText("Not synced yet")).toBeInTheDocument();
+ expect(screen.queryByText("Draft")).not.toBeInTheDocument();
});
});
});
diff --git a/frontend/src/metabase-types/api/data-app.ts b/frontend/src/metabase-types/api/data-app.ts
index 899524da0ef..b3337375b8a 100644
--- a/frontend/src/metabase-types/api/data-app.ts
+++ b/frontend/src/metabase-types/api/data-app.ts
@@ -1,13 +1,13 @@
export type DataAppId = number;
/**
- * A data app materialized from the connected repository. The repo's
- * `data_apps/<dir>/data_app.yaml` files are discovered on sync; one row exists
- * per app and the bundle is cached in the app DB and served at `/apps/:name`.
- * The repository itself is configured via the remote-sync settings.
+ * A data app: its manifest fields, plus a bundle cached in the app DB and
+ * served at `/apps/:name`.
*/
export interface DataApp {
id: DataAppId;
+ /** Stable identifier shared by the app's copies across instances. */
+ entity_id: string;
/** The app's slug. */
name: string;
display_name: string;
@@ -20,10 +20,12 @@ export interface DataApp {
* regular users never receive an outdated app.
*/
outdated: boolean;
- /** Path within the repo to the built bundle. */
+ /** Path to the built bundle, relative to the app's directory. */
bundle_path: string;
/** Admin toggle. When false the app is not served. */
enabled: boolean;
+ /** Whether the app is a draft that reserves its slug before the app is created. */
+ draft: boolean;
/** The collection that contains this app's saved questions and models. */
resource_collection_id: number | null;
/** The group that grants users access to this data app. */
@@ -33,19 +35,13 @@ export interface DataApp {
/** Whether any app member lacks access to a table used by this app. */
has_user_permission_warnings?: boolean;
/**
- * External origins the app's sandboxed bundle may `fetch`/XHR, from its
- * `data_app.yaml`. Empty means none (Metabase data still flows through the
- * SDK). Each entry is an origin, optionally with a `*.` wildcard.
+ * External origins the app's sandboxed bundle may `fetch`/XHR. Empty means
+ * none (Metabase data still flows through the SDK). Each entry is an origin,
+ * optionally with a `*.` wildcard.
*/
allowed_hosts: string[];
- /** SHA-256 of the cached bundle; `null` until the first successful sync. */
+ /** SHA-256 of the cached bundle; `null` while the app has none. */
bundle_hash: string | null;
- /** Git commit the cached bundle was synced from; `null` until first sync. */
- last_synced_sha: string | null;
- /** ISO timestamp of the last successful sync; `null` if never synced. */
- last_synced_at: string | null;
- /** Message from this app's most recent failed sync; `null` when it succeeded. */
- sync_error: string | null;
created_at: string;
updated_at: string;
}
diff --git a/frontend/src/metabase-types/api/mocks/data-app.ts b/frontend/src/metabase-types/api/mocks/data-app.ts
index e19d7a03e20..12d9c48033f 100644
--- a/frontend/src/metabase-types/api/mocks/data-app.ts
+++ b/frontend/src/metabase-types/api/mocks/data-app.ts
@@ -2,22 +2,21 @@ import type { DataApp } from "metabase-types/api";
export const createMockDataApp = (opts?: Partial<DataApp>): DataApp => ({
id: 1,
+ entity_id: "mOcKdAtAaPpEnTiTyIdXx",
name: "sales",
display_name: "Sales",
description: null,
version: 1,
outdated: false,
- bundle_path: "data_apps/sales/dist/index.js",
+ bundle_path: "dist/index.js",
enabled: true,
+ draft: false,
resource_collection_id: 1,
permission_group_id: 1,
table_ids: [],
has_user_permission_warnings: false,
allowed_hosts: [],
bundle_hash: "abc123",
- last_synced_sha: "0123456789abcdef",
- last_synced_at: "2024-01-01T00:00:00Z",
- sync_error: null,
created_at: "2024-01-01T00:00:00Z",
updated_at: "2024-01-01T00:00:00Z",
...opts,
diff --git a/resources/migrations/065/20261001_data_app_serialization.yaml b/resources/migrations/065/20261001_data_app_serialization.yaml
new file mode 100644
index 00000000000..9b9a43a459b
--- /dev/null
+++ b/resources/migrations/065/20261001_data_app_serialization.yaml
@@ -0,0 +1,121 @@
+## Data apps become regular serdes entities. They get an entity_id so serialization and remote sync can key them on a
+## stable, portable identifier; the backfill mirrors 20260911_glossary_entity_id: all three dialect variants in one
+## changeset, then NOT NULL and UNIQUE. Imports now fail as a whole rather than per app, so the per-app sync
+## columns go away.
+
+databaseChangeLog:
+ - changeSet:
+ id: v65.2026-10-01T00:00:00
+ author: ranquild
+ comment: Add entity_id column to data_app table
+ changes:
+ - addColumn:
+ tableName: data_app
+ columns:
+ - column:
+ name: entity_id
+ type: char(21)
+ remarks: Random NanoID tag for unique identity.
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: data_app
+ columnName: entity_id
+
+ - changeSet:
+ id: v65.2026-10-01T00:00:01
+ author: ranquild
+ comment: Backfill NULL data_app.entity_id values
+ changes:
+ - sql:
+ dbms: postgresql
+ sql: UPDATE data_app SET entity_id = substr(replace(gen_random_uuid()::text, '-', ''), 1, 21) WHERE entity_id IS NULL;
+ - sql:
+ dbms: mysql,mariadb
+ sql: UPDATE data_app SET entity_id = substr(replace(uuid(), '-', ''), 1, 21) WHERE entity_id IS NULL;
+ - sql:
+ dbms: h2
+ sql: UPDATE data_app SET entity_id = substr(replace(cast(random_uuid() as varchar), '-', ''), 1, 21) WHERE entity_id IS NULL;
+ rollback: []
+
+ - changeSet:
+ id: v65.2026-10-01T00:00:02
+ author: ranquild
+ comment: entity_id is NOT NULL on data_app
+ changes:
+ - addNotNullConstraint:
+ tableName: data_app
+ columnName: entity_id
+ columnDataType: char(21)
+
+ - changeSet:
+ id: v65.2026-10-01T00:00:03
+ author: ranquild
+ comment: entity_id is unique on data_app
+ changes:
+ - addUniqueConstraint:
+ tableName: data_app
+ columnNames: entity_id
+ constraintName: uq_data_app_entity_id
+ rollback:
+ - dropUniqueConstraint:
+ tableName: data_app
+ constraintName: uq_data_app_entity_id
+
+ - changeSet:
+ id: v65.2026-10-01T00:00:04
+ author: ranquild
+ comment: Drop data_app.sync_error, which only the removed per-app sync wrote
+ changes:
+ - dropColumn:
+ tableName: data_app
+ columnName: sync_error
+ rollback:
+ - addColumn:
+ tableName: data_app
+ columns:
+ - column:
+ name: sync_error
+ type: ${text.type}
+ remarks: Error from the most recent failed sync of this app, NULL when the last sync succeeded
+ constraints:
+ nullable: true
+
+ - changeSet:
+ id: v65.2026-10-01T00:00:05
+ author: ranquild
+ comment: Drop data_app.last_synced_sha, which only the removed per-app sync wrote
+ changes:
+ - dropColumn:
+ tableName: data_app
+ columnName: last_synced_sha
+ rollback:
+ - addColumn:
+ tableName: data_app
+ columns:
+ - column:
+ name: last_synced_sha
+ type: varchar(40)
+ remarks: Git commit SHA the app was last synced from
+ constraints:
+ nullable: true
+
+ - changeSet:
+ id: v65.2026-10-01T00:00:06
+ author: ranquild
+ comment: Drop data_app.last_synced_at, which only the removed per-app sync wrote
+ changes:
+ - dropColumn:
+ tableName: data_app
+ columnName: last_synced_at
+ rollback:
+ - addColumn:
+ tableName: data_app
+ columns:
+ - column:
+ name: last_synced_at
+ type: ${timestamp_type}
+ remarks: Timestamp of the last successful sync of this app
+ constraints:
+ nullable: true
diff --git a/skills/metabase-data-app-migrate/SKILL.md b/skills/metabase-data-app-migrate/SKILL.md
index f368170018c..f8f1eee6e40 100644
--- a/skills/metabase-data-app-migrate/SKILL.md
+++ b/skills/metabase-data-app-migrate/SKILL.md
@@ -143,7 +143,7 @@ Do not batch steps across upgrades. Do not touch `version` before item 5.
```bash
( source "$ROOT/.env.local"; curl -s -H "x-api-key: $DATA_APP_MB_API_KEY" "$DATA_APP_MB_URL/api/apps/<slug>" )
```
- must show `"version": M`, `"outdated": false`, and `"sync_error": null`.
+ must show `"version": M` and `"outdated": false`.
A response holding only `name` and `display_name` means the key is not an
admin's: ask for one, or ask the user to read the admin list, and say the
proof is still pending. Report the command and the result. Do not claim the
diff --git a/skills/metabase-data-app-migrate/references/upgrade-guide-template.md b/skills/metabase-data-app-migrate/references/upgrade-guide-template.md
index f0735c940c1..228ac28ec35 100644
--- a/skills/metabase-data-app-migrate/references/upgrade-guide-template.md
+++ b/skills/metabase-data-app-migrate/references/upgrade-guide-template.md
@@ -91,7 +91,7 @@ ends with a **Done when** that reads the API:>
Done when:
```bash
-( source "$ROOT/.env.local"; curl -s -H "x-api-key: $DATA_APP_MB_API_KEY" "$DATA_APP_MB_URL/api/apps/<slug>" ) | grep -q '"sync_error":null' && echo ok
+( source "$ROOT/.env.local"; curl -s -H "x-api-key: $DATA_APP_MB_API_KEY" "$DATA_APP_MB_URL/api/apps/<slug>" ) | grep -q '"outdated":false' && echo ok
```
## Done-check summary
diff --git a/skills/metabase-data-app-setup/SKILL.md b/skills/metabase-data-app-setup/SKILL.md
index 9d8900b439d..84674b6d6af 100644
--- a/skills/metabase-data-app-setup/SKILL.md
+++ b/skills/metabase-data-app-setup/SKILL.md
@@ -7,7 +7,7 @@ description: Scaffold a new Metabase data-app into the connected remote-sync rep
A Metabase **data-app** is a single JS bundle that the host loads inside a Near Membrane sandbox and renders inside its own React tree. The scaffold is a Vite + React + TypeScript project: source under `src/`, a dev server that previews the app against a real Metabase **through the same Near Membrane sandbox + distortion rules Metabase uses in production** — so `npm run dev` behaves like production, including for third-party libraries the app bundles — and `npm run build` producing a single `dist/index.js`. (Because the sandbox runs a built bundle, a change rebuilds it and does a *soft reload* — re-evaluates the bundle in the sandbox and remounts the app, keeping auth/SDK loaded — rather than hot-swapping modules; component state resets, but there's no full browser refresh.) The dev preview also shows a corner **⚠ Diagnostics** toolbar that captures runtime errors — including the sandbox's otherwise-opaque blocked-API messages — so failures surface instead of being swallowed. The same data is served as JSON at `http://localhost:5174/__data-app/diagnostics`, which is how *you* read it (see "Reading the diagnostics feed" below) — you have a shell, not a browser, and these failures are invisible from the terminal otherwise.
-**Data apps are served from Git, not uploaded.** A single repository is connected to Metabase via remote-sync (Admin → Settings → Remote sync). Each app lives in its own directory `data_apps/<app>/` inside that repo — its source, a `data_app.yaml` (name/path), and the committed built bundle at the `path` its `data_app.yaml` declares (`dist/index.js` by default). On each remote-sync import Metabase materializes one app per directory and serves it at `/apps/<slug>` url, where the slug **is** the directory's name. So this skill always scaffolds **into the connected repo's `data_apps/<app>/` directory**, never as a standalone project.
+**Data apps are served from Git** (they can also be created and updated through `/api/apps`). A single repository is connected to Metabase via remote-sync (Admin → Settings → Remote sync). Each app lives in its own directory `data_apps/<slug>/` inside that repo — its source, a `data_app.yaml` (slug/name/path), and the committed built bundle at the `path` its `data_app.yaml` declares (`dist/index.js` by default). On each remote-sync import Metabase materializes each app and serves it at `/apps/<slug>`, where the slug is the `slug` its `data_app.yaml` declares. So this skill always scaffolds **into the connected repo's `data_apps/<slug>/` directory**, never as a standalone project.
**The scaffold ships inside this skill at `./template/`** — a Vite + React + TypeScript project that was installed alongside the skill. Step 3 just copies it into the app directory; the skill then guides you through the customization + first-app-content steps — it never generates project files from scratch. If you find yourself writing `package.json`, `vite.config.ts`, `tsconfig.json`, or `src/index.tsx` by hand, stop — copy the template instead.
@@ -33,7 +33,7 @@ Data apps live inside the Git repository connected to Metabase via remote-sync.
## Step 2 — Name the app and create its directory
-1. Settle on the app's **directory name** before scaffolding — it is used verbatim as the slug (the `/apps/<slug>` URL), so it **must be dash-cased**: lowercase letters, numbers, and single dashes (`[a-z0-9]+(?:-[a-z0-9]+)*`), e.g. `sales-overview`. Anything else (uppercase, spaces, underscores) is rejected on sync. If the purpose isn't clear yet, ask a one-line "what's this app for?" and propose a name; confirm it.
+1. Settle on the app's **slug** before scaffolding — the `/apps/<slug>` URL — so it **must be dash-cased**: lowercase letters, numbers, and single dashes (`[a-z0-9]+(?:-[a-z0-9]+)*`), e.g. `sales-overview`. Anything else (uppercase, spaces, underscores) is rejected on sync. If the purpose isn't clear yet, ask a one-line "what's this app for?" and propose a slug; confirm it.
2. Ensure `<repo>/data_apps/` exists; create it if missing.
3. Create `<repo>/data_apps/<slug>/`. **If it already exists**, treat it as an existing project (see below) — never overwrite without confirmation.
@@ -131,16 +131,22 @@ Once the template is in `<repo>/data_apps/<slug>/` (run everything below from th
6. **The lockfile and the built bundle must both be committed.** Metabase serves the file at the `path` declared in `data_app.yaml` (the template builds to `dist/index.js`) straight from the committed Git tree, and the lockfile keeps installs reproducible. **Verify with `git status`** after `npm install` and a build: both must appear as committable files.
7. `npm run dev` and confirm the preview at http://localhost:5174 renders the starter "Hello, data app" message.
8. If the preview hits CORS, add `http://localhost:5174` under Admin → Embedding → Embedded analytics SDK → CORS.
-9. **Edit `data_app.yaml`** (it ships with the template, in the app directory). This is the per-app config Metabase reads on sync — one file per app. Fill in its fields for this app:
+9. **Edit `data_app.yaml`** (it ships with the template, in the app directory). This is the per-app config Metabase reads on sync — one file per app. Generate a new entity id with `node -e "console.log(require('crypto').randomBytes(16).toString('base64url').slice(0, 21))"` and fill in its fields for this app:
```yaml
+ version: 1 # data app contract version — leave as-is (see below)
name: Sales App # display name shown in the admin UI
+ slug: sales-app # the /apps/<slug> URL
description: Pipeline health and quota attainment by region # optional — see below
- version: 1 # data app contract version — leave as-is (see below)
path: ./dist/index.js # bundle path, relative to this app's directory — leave as-is unless you change the build output
# allowed_hosts: # optional — external origins the app may fetch/XHR (see below)
# - https://api.example.com
# - https://*.internal.acme.com
+ entity_id: Xq2v9LbN0mTz4wRk7YsJd # the generated entity id
+ serdes/meta:
+ - model: DataApp
+ id: Xq2v9LbN0mTz4wRk7YsJd
+ label: sales-app
```
Commit it alongside the built bundle (the file `path` points at).
@@ -524,7 +530,7 @@ Do not wrap `InteractiveQuestion` or `StaticQuestion` in containers that clip or
## Sync to Metabase
-Data apps are delivered by Git, not uploaded — you commit the app directory and Metabase pulls it on its next remote-sync import.
+Data apps are delivered by Git — you commit the app directory and Metabase pulls it on its next remote-sync import.
1. `npm run build` → produces the bundle at your `data_app.yaml` `path` (the template builds to `dist/index.js`).
2. From the **repo root**, commit the app directory — its `data_app.yaml`, the built bundle (the file `path` points at), the source, and the lockfile — and **push**:
@@ -538,7 +544,7 @@ Data apps are delivered by Git, not uploaded — you commit the app directory an
> **Don't offer to "deploy" the app or ask how the bundle reaches a staging environment** — there is no separate deploy step, and the question only confuses users: Metabase imports the committed bundle straight from the connected repo on its next sync. Once the change is on the branch Metabase syncs from — however the user gets it there (a merged PR, or a push straight to that branch) — just tell them to pull it in and open the app in Metabase at `/apps/<slug>`.
- **To update:** commit a new build and pull again.
-- **To remove:** delete the app's directory from the repo and push — the next sync removes it. You can't delete a repo-managed app from the UI; the *Remove* action on the Data apps admin page appears only after the repo is disconnected, to clear out apps left behind.
+- **To remove:** delete the app's directory from the repo and push — the next sync removes it.
## Common pitfalls
diff --git a/skills/metabase-data-app-setup/template/README.md b/skills/metabase-data-app-setup/template/README.md
index d07d1c385ec..941b35212d8 100644
--- a/skills/metabase-data-app-setup/template/README.md
+++ b/skills/metabase-data-app-setup/template/README.md
@@ -4,10 +4,10 @@ A Metabase **data app** — a single-bundle React app (built with the Embedding
SDK) that Metabase renders inside an isolated, sandboxed iframe at
`/apps/<slug>`.
-Data apps are delivered through **Git, not uploaded**: this directory lives at
+Data apps are delivered through **Git**: this directory lives at
`data_apps/<slug>/` inside a repository connected to Metabase via remote sync.
You commit the built bundle (`dist/index.js`), and on the next remote-sync
-import Metabase materializes the app and serves it. There is no upload step.
+import Metabase materializes the app and serves it.
## Develop
diff --git a/skills/metabase-data-app-setup/template/data_app.yaml b/skills/metabase-data-app-setup/template/data_app.yaml
index 43fb1b9b106..9fae16f682d 100644
--- a/skills/metabase-data-app-setup/template/data_app.yaml
+++ b/skills/metabase-data-app-setup/template/data_app.yaml
@@ -2,6 +2,8 @@ version: 1
name: Data App Template
+slug: data-app-template
+
description: A summary of what this app does
path: ./dist/index.js
@@ -12,3 +14,10 @@ path: ./dist/index.js
# allowed_hosts:
# - https://api.example.com
# - https://*.internal.acme.com
+
+entity_id: replace-entity-id-xxx
+
+serdes/meta:
+- model: DataApp
+ id: replace-entity-id-xxx
+ label: data-app-template
diff --git a/src/metabase/internal_stats/data_apps.clj b/src/metabase/internal_stats/data_apps.clj
index 458bf9b8499..ad38a434bac 100644
--- a/src/metabase/internal_stats/data_apps.clj
+++ b/src/metabase/internal_stats/data_apps.clj
@@ -3,7 +3,6 @@
[metabase.internal-stats.db :as internal-stats.db]))
(defn data-app-stats
- "How many data apps this instance actually serves: enabled and synced without
- error (`sync_error IS NULL`), matching what the UI treats as an openable app."
+ "How many data apps this instance actually serves: enabled ones that aren't drafts."
[]
{:data-app-count (internal-stats.db/enabled-data-app-count)})
diff --git a/src/metabase/internal_stats/db.clj b/src/metabase/internal_stats/db.clj
index 2c6a14a12ef..dc50df1866c 100644
--- a/src/metabase/internal_stats/db.clj
+++ b/src/metabase/internal_stats/db.clj
@@ -10,9 +10,9 @@
[toucan2.core :as t2]))
(mu/defn enabled-data-app-count
- "The number of enabled data apps without a sync error."
+ "The number of enabled data apps that aren't drafts."
[]
- (t2/count :data_app :enabled true :sync_error nil))
+ (t2/count :data_app :enabled true :draft false))
(mu/defn embedded-dashboard-count
"The number of unarchived Dashboards with embedding enabled."
diff --git a/src/metabase/models/db.clj b/src/metabase/models/db.clj
index ed625f5a540..170371c63b1 100644
--- a/src/metabase/models/db.clj
+++ b/src/metabase/models/db.clj
@@ -60,6 +60,7 @@
:model/DashboardCard :metabase.dashboards.schema/dashboard-card.update
:model/DashboardCardSeries :metabase.dashboards.schema/dashboard-card-series.update
:model/DashboardTab :metabase.dashboards.schema/dashboard-tab.update
+ :model/DataApp :metabase-enterprise.data-apps.schema/data-app.update
:model/Database :metabase.warehouses.schema/database.update
:model/Dimension :metabase.warehouse-schema.schema/dimension.update
:model/Document :metabase.documents.schema/document.update
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index c4277b713fe..0834dbd6c30 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -587,6 +587,14 @@
[entity]
(:serdes/meta entity))
+(defmulti resource-paths
+ "Paths of the `:serdes/resources` stored next to an ingested entity's YAML file, relative to its directory."
+ {:arglists '([ingested])}
+ ingested-model)
+
+(defmethod resource-paths :default [_]
+ nil)
+
(defmulti load-find-local
"Given a path, tries to look up any corresponding local entity.
@@ -764,8 +772,10 @@
[{:label (:name entity) :key (:entity_id entity)}]])))
(defmulti storage-path
- "Returns a vector of maps with `:label` and optional `:key` for each path segment.
- `:label` is the human-readable name; `:key` is a deduplication identity (entity_id, name, or nil).
+ "Returns a vector of maps with `:label` and optional `:key` and `:style` for each path segment.
+ `:label` is the human-readable name; `:key` is a deduplication identity (entity_id, name, or nil); `:style` is
+ `:name` (the default) for a label that is slugified into a file name, or `:slug` for a label that is already a slug
+ safe to use as a file name as is.
Dispatches on model name."
{:arglists '([entity ctx])}
(fn [entity _] (ingested-model entity)))
diff --git a/test/metabase/internal_stats/data_apps_test.clj b/test/metabase/internal_stats/data_apps_test.clj
index e0e9505bf3e..9d946185d47 100644
--- a/test/metabase/internal_stats/data_apps_test.clj
+++ b/test/metabase/internal_stats/data_apps_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer [deftest is testing]]
[metabase.internal-stats.data-apps :as sut]
[metabase.test :as mt]
+ [metabase.util :as u]
[toucan2.core :as t2]))
(defn- data-app!
@@ -11,6 +12,7 @@
[& {:as extra}]
(let [app-name (mt/random-name)]
(t2/insert! :data_app (merge {:name app-name
+ :entity_id (u/generate-nano-id)
:display_name "App"
:bundle_path "data_apps/app/index.js"
:created_at :%now
@@ -31,8 +33,8 @@
(testing "an app the admin disabled is out of service, so it doesn't count"
(create! :enabled false)
(is (= 2 (- (:data-app-count (sut/data-app-stats)) before))))
- (testing "an app that failed to sync is not served, so it doesn't count even when enabled"
- (create! :sync_error "boom")
+ (testing "a draft is not served, so it doesn't count even when enabled"
+ (create! :draft true)
(is (= 2 (- (:data-app-count (sut/data-app-stats)) before)))))
(finally
(t2/delete! :data_app :name [:in @app-names])))))
diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.clj
index c91bd6f7d22..6af3a2686ea 100644
--- a/test/metabase/test/util.clj
+++ b/test/metabase/test/util.clj
@@ -179,6 +179,12 @@
:name (u.random/random-name)
:position 0}))
+ :model/DataApp
+ (fn [_] {:entity_id (u/generate-nano-id)
+ :name (u/lower-case-en (u.random/random-name))
+ :display_name (u.random/random-name)
+ :bundle_path "dist/index.js"})
+
:model/Database
(fn [_] (default-timestamped
{:details {}
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index 9d6ebe5bc1b..e6c3e1ca093 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -103,7 +103,8 @@
(defn- table-details [table]
(-> (merge (mt/obj->json->obj (mt/object-defaults :model/Table))
(select-keys table [:active :created_at :db_id :description :display_name :entity_type
- :id :name :rows :schema :updated_at :visibility_type :initial_sync_status]))
+ :id :name :rows :schema :updated_at :visibility_type :initial_sync_status
+ :is_writable]))
(update :entity_type #(when % (str "entity/" (name %))))
(update :visibility_type #(when % (name %)))
(update :schema str)))