The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add data app generate endpoints for a new app, its schema, and its collection's files (#83992)
Alexander Polyankin9f28c701e7e9
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex ea8780f83c2..4254c71a1b4 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1468,7 +1468,6 @@metabase.metabot.settingsmetabase.metabot.toolsmetabase.metabot.tools.construct- metabase.metabot.tools.entity-detailsmetabase.metabot.tools.search}:module-exports #{metabot.agent-apimetabot.llm@@ -3090,39 +3089,6 @@:model/TransformRun:model/TransformTag}}- typed-schemas- {:team "Embedding"- :api #{metabase.typed-schemas.core}- :uses #{actions- audit-app- collections- lib- lib.be- lib.schema- lib.types- metabot- metrics- permissions- premium-features- queries- system- util- warehouse-schema-overlay}- :model-imports #{:model/Action- :model/Card- :model/Collection- :model/Database- :model/Field- :model/Measure- :model/QueryAction- :model/Table}}-- typed-schemas.rest- {:team "Embedding"- :ns-prefix "metabase.typed-schemas-rest"- :api #{metabase.typed-schemas-rest.api}- :uses #{api typed-schemas util}}-upload{:team "Graphy":api #{metabase.upload.api@@ -3805,6 +3771,7 @@:uses #{actionsapiapi-scope+ audit-appcollectionsdriverevents@@ -3812,12 +3779,17 @@lib.belib.metadatalib.schema+ lib.types+ metabot+ metricsmodelspermissionspremium-features+ queriesremote-syncrequestsettings+ systemutilwarehouse-schema-overlayenterprise/serialization}@@ -3827,6 +3799,7 @@:model/DataPermissions:model/Database:model/Field+ :model/Measure:model/Permissions:model/QueryAction:model/Table}diff --git a/bin/embedding-sdk/test-sdk-package.mjs b/bin/embedding-sdk/test-sdk-package.mjsindex 9af0d452420..fb97651519a 100644--- a/bin/embedding-sdk/test-sdk-package.mjs+++ b/bin/embedding-sdk/test-sdk-package.mjs@@ -71,7 +71,7 @@ try {NPM_COMMAND_OPTIONS,);- for (const command of ["print-resources", "check-resources"]) {+ for (const command of ["write-resources", "check-resources"]) {const output = execFileSync(process.execPath,[diff --git a/e2e/support/helpers/e2e-data-app-helpers.ts b/e2e/support/helpers/e2e-data-app-helpers.tsindex 4bcc6705ab5..e8dfb286470 100644--- a/e2e/support/helpers/e2e-data-app-helpers.ts+++ b/e2e/support/helpers/e2e-data-app-helpers.ts@@ -420,40 +420,59 @@ const resourceCard = ({});/**- * The copies of the actions `copies` name, as an author writes them into the- * app's collection: what Metabase serializes for each source action, with the- * copy's entity ID and in the app's `collection`.+ * What `POST /api/apps/generate/resources` answers for the actions `copies` name: the+ * file of each copy, by position, for the app collection `collection`.*/-export function serializeDataAppActionCopies(+export function serializeDataAppActions(copies: Array<{ sourceActionId: number; entityId: string }>,collection: string,) {return cy- .request<{ actions: Array<{ entity: ResourceEntity }> }>(- "POST",- "/api/apps/serialize-resources",- {- collection,- actions: copies.map(({ sourceActionId }) => sourceActionId),- },- )+ .request<{+ actions: Array<{ file: string; yaml: string } | { error: string }>;+ }>("POST", "/api/apps/generate/resources", {+ actions: copies.map(({ sourceActionId, entityId }) => ({+ action_id: sourceActionId,+ entity_id: entityId,+ collection_id: collection,+ })),+ }).then(({ body }) =>- cy.wrap(- copies.map(({ entityId }, index): ResourceEntity => {- const { entity } = body.actions[index];-- return {- ...entity,- entity_id: entityId,- collection_id: collection,- "serdes/meta": serdesMeta("Action", entityId, String(entity.name)),- };- }),- { log: false },- ),+ body.actions.map((answer) => {+ if ("error" in answer) {+ throw new Error(answer.error);+ }++ return answer;+ }),);}+/**+ * The copies of the actions `copies` name, as an author writes them into the+ * app's collection: what Metabase serializes for each source action, with the+ * copy's entity ID, in the app's `collection`.+ */+export function serializeDataAppActionCopies(+ copies: Array<{ sourceActionId: number; entityId: string }>,+ collection: string,+) {+ return serializeDataAppActions(copies, collection).then((files) =>+ cy.wrap(+ files.map(({ yaml: text }): ResourceEntity => {+ const entity = yaml.load(text);++ if (!isObject(entity)) {+ throw new Error(`Generated an action copy that isn't a map: ${text}`);+ }++ return entity;+ }),+ { log: false },+ ),+ );+}+/*** The YAML an author writes for an app's collection, in the Metabase* representation format: plain data for `writeDataAppResources`, read from@@ -559,7 +578,7 @@ export function declareDataAppQueries(/*** Runs the data app CLI the host app has installed, the one an author runs:- * `embedding-sdk-react data-apps <command>`. `check-resources` never calls Metabase; `print-resources`+ * `embedding-sdk-react data-apps <command>`. `check-resources` never calls Metabase; `write-resources`* reaches it through `env` (see `dataAppCliEnv`).*/export function runDataAppCli(command: string, env?: Record<string, string>) {diff --git a/e2e/test-host-app/data-apps/resources.cy.spec.ts b/e2e/test-host-app/data-apps/resources.cy.spec.tsindex dffb44fb74a..8e3f9501e2f 100644--- a/e2e/test-host-app/data-apps/resources.cy.spec.ts+++ b/e2e/test-host-app/data-apps/resources.cy.spec.ts@@ -1,3 +1,5 @@+import yaml from "js-yaml";+import { USERS } from "e2e/support/cypress_data";import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";import type { PortableTable } from "e2e/support/helpers";@@ -60,47 +62,42 @@ describe("Embedding SDK: data-app resources (queries)", () => {});describe("the CLI", () => {- it("prints the query Metabase builds from a definition, as an author writes the saved question from it", () => {+ it("writes the saved question Metabase builds from a definition into the collection's folder", () => {const question = H.newEntityId();H.declareDataAppQueries(APP_ROOT(), [{ name: "Orders", tableId: ORDERS_ID, savedQuestionEntityId: question },]);+ H.writeDataAppResources(APP_ROOT(), { collection: collection() });H.dataAppCliEnv().then((env) =>- H.runDataAppCli("print-resources", env).then(({ exitCode, stdout }) => {+ H.runDataAppCli("write-resources", env).then(({ exitCode, stdout }) => {expect(exitCode, stdout).to.eq(0);- const printed = JSON.parse(stdout);- expect(printed).to.deep.include({- actions: [],- metrics: [],- });- const [query] = printed.queries;- expect(query).to.deep.include({- export: "Orders",- file: "queries/orders.query.ts",- savedQuestionEntityId: question,- metrics: [],- });- // The saved question is complete: the author writes it as it is.- expect(query.entity).to.deep.include({- entity_id: question,- collection_id: COLLECTION,- name: "Orders",- type: "question",- display: "table",- dataset_query: {- "lib/type": "mbql/query",- database: "Sample Database",- stages: [- {- "lib/type": "mbql.stage/mbql",- "source-table": ["Sample Database", "PUBLIC", "ORDERS"],- },- ],- },- "serdes/meta": [{ model: "Card", id: question, label: "orders" }],+ const [, written] = stdout.match(/^Wrote (.+)$/m) ?? [];+ expect(written).to.match(+ /^collections\/data_apps\/data_app__[^\/]+\/orders\.yaml$/,+ );+ expect(stdout.trim().split("\n")).to.have.length(1);++ cy.readFile(`${APP_ROOT()}/${written}`).then((text) => {+ expect(yaml.load(text)).to.deep.include({+ entity_id: question,+ collection_id: COLLECTION,+ name: "Orders",+ type: "question",+ display: "table",+ dataset_query: {+ "lib/type": "mbql/query",+ database: "Sample Database",+ stages: [+ {+ "lib/type": "mbql.stage/mbql",+ "source-table": ["Sample Database", "PUBLIC", "ORDERS"],+ },+ ],+ },+ "serdes/meta": [{ model: "Card" }],+ });});- expect(query.entity.creator_id).to.be.a("string");}),);});diff --git a/e2e/test/scenarios/data-apps/resources-production-actions.cy.spec.ts b/e2e/test/scenarios/data-apps/resources-production-actions.cy.spec.tsindex e8de43d1a50..6cae23a578b 100644--- a/e2e/test/scenarios/data-apps/resources-production-actions.cy.spec.ts+++ b/e2e/test/scenarios/data-apps/resources-production-actions.cy.spec.ts@@ -1,3 +1,5 @@+import yaml from "js-yaml";+import { USERS, WRITABLE_DB_ID } from "e2e/support/cypress_data";const { H } = cy;@@ -87,16 +89,22 @@ describe(},});- cy.request("POST", "/api/apps/serialize-resources", {- collection: COLLECTION,- actions: [source.id],- }).then(({ body }) => {- expect(body.actions[0].entity).to.deep.include({- entity_id: source.entity_id,+ const copy = H.newEntityId();++ H.serializeDataAppActions(+ [{ sourceActionId: source.id, entityId: copy }],+ COLLECTION,+ ).then(([file]) => {+ expect(file.file).to.eq("add_team.yaml");++ const entity = yaml.load(file.yaml);+ expect(entity).to.deep.include({+ entity_id: copy,+ collection_id: COLLECTION,type: "query",});- expect(body.actions[0].entity).not.to.have.property("model_id");- expect(body.actions[0].entity)+ expect(entity).not.to.have.property("model_id");+ expect(entity).to.have.nested.property("visualization_settings.fields.score").that.deep.include({ hidden: true, defaultValue: 0 });});diff --git a/e2e/test/scenarios/data-apps/semantic-schema.cy.spec.ts b/e2e/test/scenarios/data-apps/semantic-schema.cy.spec.tsindex ba566f9e521..99a2c61f7f3 100644--- a/e2e/test/scenarios/data-apps/semantic-schema.cy.spec.ts+++ b/e2e/test/scenarios/data-apps/semantic-schema.cy.spec.ts@@ -44,10 +44,7 @@ describe("scenarios > data apps > semantic schema", () => {});it("tells the author about the filter a metric applies", () => {- cy.request<string>(- "GET",- "/api/typed-schemas/v1/typescript?include-metric-library=true",- ).then(({ body }) => {+ cy.request<string>("GET", "/api/apps/generate/schemas").then(({ body }) => {const metadata = H.typedSchemaMetadata(body, "revenue: {");expect(metadata.description).to.eq(METRIC_DESCRIPTION);diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/README.md b/enterprise/backend/src/metabase_enterprise/data_apps/README.mdindex 98c156117fc..0b5455a521c 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/README.md+++ b/enterprise/backend/src/metabase_enterprise/data_apps/README.md@@ -89,9 +89,15 @@ assets (`metabase.server.routes/static-files-handler`).- `PUT /api/apps/:slug` — update manifest fields or the bundle, or toggle `enabled` (superuser).- `DELETE /api/apps/:slug` — drop a row, its bundle, and its owned resources (superuser).- `GET /api/apps/repo-status` — whether a repo is connected (superuser).-- `POST /api/apps/serialize-resources` — what the files of an app's collection are written from: the query Metabase builds- from each `defineQuery` definition, and the actions and metrics it copies, all as serialization writes- them (`resource_serialization.clj`). An action must belong to no model (superuser).+- `POST /api/apps/generate/app` — a new app's `data_app.yaml` and its collection's file, each at its path+ from the repository root, with new entity IDs (`generate/app.clj`; superuser).+- `GET /api/apps/generate/schemas` — the TypeScript module an app's definitions are written against: the+ root libraries' tables and metrics, and the query actions that belong to no model+ (`generate/schemas.clj`; superuser).+- `POST /api/apps/generate/resources` — the files of an app's collection, each a file name and the YAML a+ remote-sync export writes: a saved question per `defineQuery` definition, a copy of each action, and a+ copy of each metric the queries aggregate (`generate/resources.clj`). An action must belong to no model+ (superuser).Responses are field-filtered by role: superusers get full metadata, everyone else gets `name` and`display_name` only. The bundle blob is never serialized into JSON, and metadata reads go through@@ -140,7 +146,11 @@ The Data Apps feature is required for all app API endpoints, including group lis| `api.clj` | The `/api/apps` endpoints, bundle serving, ETag handling. || `resources.clj` | Lifecycle of the app-owned collection and derived collection permissions. || `models/data_app.clj` | The `:model/DataApp` Toucan model: hooks, permissions, default fields, serialization. |-| `resource_serialization.clj` | The serialization an app's resource files are written from: built queries, actions, metrics. |+| `generate.clj` | The facade over what Metabase generates for an app's author: its files, its schema, its collection's files. |+| `generate/app.clj` | A new app's `data_app.yaml` and collection file, each at its path with its YAML. |+| `generate/resources.clj` | The files of an app's collection: built queries, action copies, metric copies, each with its YAML. |+| `generate/schemas.clj` | The TypeScript schema of the root libraries' tables and metrics and the model-less query actions. |+| `generate/schemas/` | The schema's stages: data access (`source`), entities (`table`, `metric`, `action`, `common`), rendering (`render`, `javascript`). || `query_definition.clj` | The closed schema of a `defineQuery` definition the serialization accepts. || `resource_validation.clj` | What the files of an app's collection may hold, checked on the whole snapshot before an import. || `resource_tables.clj` | The tables an app's resources read, recorded on the app after an import. |diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/api.clj b/enterprise/backend/src/metabase_enterprise/data_apps/api.cljindex 557c4c9b405..d36921458a4 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/api.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/api.clj@@ -11,10 +11,9 @@[metabase-enterprise.data-apps.apps :as data-apps.apps][metabase-enterprise.data-apps.config :as data-app.config][metabase-enterprise.data-apps.db :as data-apps.db]+ [metabase-enterprise.data-apps.generate :as data-apps.generate][metabase-enterprise.data-apps.group-access :as data-app.group-access][metabase-enterprise.data-apps.models.data-app :as data-app]- [metabase-enterprise.data-apps.query-definition :as query-definition]- [metabase-enterprise.data-apps.resource-serialization :as data-app.resource-serialization][metabase-enterprise.data-apps.schema :as data-apps.schema][metabase.api-scope.data-app :as api-scope][metabase.api.common :as api]@@ -35,8 +34,8 @@;;; ------------------------------------------------ Constants -------------------------------------------------;; Slug must not collide with the literal `repo-status`/`sandbox-host` sub-routes.-(def ^:private slug-regex #"(?!repo-status$|sandbox-host$)[^/]+")+;; Slug must not collide with the literal `repo-status`/`sandbox-host`/`generate` sub-routes.+(def ^:private slug-regex #"(?!repo-status$|sandbox-host$|generate$)[^/]+")(def ^:private bundle-response-headers;; `no-cache` so the browser may cache but must revalidate via@@ -132,40 +131,6 @@[:configured :boolean][:url [:maybe :string]]])-(def ^:private SerializeResourcesRequest- [:map {:closed true}- [:collection ms/NanoIdString]- [:queries {:default []} [:sequential [:map {:closed true}- [:export ms/NonBlankString]- [:entity_id ms/NanoIdString]- [:query ::query-definition/query-definition]]]]- [:actions {:default []} [:sequential {:distinct true} ms/PositiveInt]]])--(def ^:private SerializedQuery- [:or- [:map {:closed true}- [:export :string]- [:entity :map]- [:metrics [:sequential :string]]]- [:map {:closed true}- [:export :string]- [:error :string]]])--(def ^:private SerializedEntity- [:or- [:map {:closed true}- [:id ms/PositiveInt]- [:entity :map]]- [:map {:closed true}- [:id ms/PositiveInt]- [:error :string]]])--(def ^:private SerializeResourcesResponse- [:map {:closed true}- [:queries [:sequential SerializedQuery]]- [:actions [:sequential SerializedEntity]]- [:metrics [:sequential SerializedEntity]]])-(def ^:private AddGroupsRequest[:map {:closed true:decode/api (fn [body]@@ -321,17 +286,49 @@;; above (returning `generic-204-no-content` would fail that validation).nil)-(api.macros/defendpoint :post "/serialize-resources" :- SerializeResourcesResponse- "Serialize what the files of a data app's collection are written from, as serialization writes it: the saved question- holding the query Metabase builds from each `defineQuery` definition in `queries`, in the app's `collection`,- the actions in `actions`, which must belong to no model, and the metrics the queries aggregate. Each item answers- on its own, with its serialization or the error that stops it. For superusers: the files are written into the app's+(api.macros/defendpoint :post "/generate/app" :- ::data-apps.schema/app-files+ "A new data app's `data_app.yaml` and its collection's file, each at its path from the repository root and with the+ YAML a remote-sync export writes, with new entity IDs. For superusers: the files are written into the app'srepository, which only an admin works with."[_route-params_query-params- {:keys [queries actions collection]} :- SerializeResourcesRequest]+ body :- ::data-apps.schema/app-request]+ (api/check-superuser)+ (data-apps.generate/app body))++(def ^:private typescript-response-headers+ {"Content-Type" "text/typescript; charset=utf-8"+ "X-Content-Type-Options" "nosniff"+ "Cross-Origin-Resource-Policy" "same-origin"+ "Referrer-Policy" "no-referrer"+ "Cache-Control" "no-store"})++(api.macros/defendpoint :get "/generate/schemas" :- :any+ "The TypeScript module a data app's definitions are written against: the root libraries' tables and metrics, and+ the query actions that belong to no model. For superusers: a data app's author builds from it, and only an admin+ works with an app's repository."+ []+ (api/check-superuser)+ {:status 200+ :headers typescript-response-headers+ :body (data-apps.generate/schemas)})++(api.macros/defendpoint :post "/generate/resources" :- [:map {:closed true}+ [:queries [:sequential ::data-apps.schema/file]]+ [:actions [:sequential ::data-apps.schema/file]]+ [:metrics [:sequential ::data-apps.schema/file]]]+ "The files of a data app's collection, each a file name in the collection's folder and the YAML a remote-sync+ export writes: the saved question of each `defineQuery` definition in `queries`, the copy of each action without a+ model in `actions`, and the copies of the metrics the queries aggregate. Each item answers on its own, with its file+ or the error that stops it. For superusers: the files are written into the app's repository, which only an admin+ works with."+ [_route-params+ _query-params+ body :- [:map {:closed true}+ [:queries {:optional true} [:maybe [:sequential ::data-apps.schema/query]]]+ [:actions {:optional true} [:maybe [:sequential ::data-apps.schema/action]]]]](api/check-superuser)- (data-app.resource-serialization/serialize-resources collection queries actions))+ (data-apps.generate/resources body))(def ^:private GroupPermissionWarning[:mapdiff --git a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj b/enterprise/backend/src/metabase_enterprise/data_apps/db.cljindex 34988055ee1..35d50f66103 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/db.clj@@ -3,6 +3,8 @@additional logic, so the rest of the module only touches `toucan2.core` for model definitions and hydration methods."(:require[metabase-enterprise.data-apps.schema :as data-apps.schema]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.queries.core :as queries][metabase.util :as u][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]@@ -282,3 +284,86 @@:join [[(t2/table-name :model/Database) :d] [:= :d.id :t.db_id]]:where [:and [:in :t.id table-ids] [:= :t.active true]]:order-by [[:d.name :asc] [:t.schema :asc] [:t.display_name :asc]]}))++(mu/defn metric-cards-in-collections+ "The non-archived metric Cards in `collection-ids`, in name then id order."+ [collection-ids :- [:set ::lib.schema.id/collection]]+ (if (seq collection-ids)+ (t2/select :model/Card {:where [:and+ [:= :type "metric"]+ [:= :archived false]+ [:in :collection_id collection-ids]]+ :order-by [[:name :asc] [:id :asc]]})+ []))++(mu/defn field-ids-and-table-ids+ "The id and Table id of the Fields with `field-ids`."+ [field-ids :- [:set ::lib.schema.id/field]]+ (t2/select [:model/Field :id :table_id] :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))++(mu/defn card-dimensions+ "The query-relevant columns (`:dimensions` and `:dimension_mappings` among them) of the Card with `card-id`,+ or nil."+ [card-id :- ::lib.schema.id/card]+ (queries/card-query-info card-id))++(mu/defn table-names+ "The id, database id, name, and display name of the Tables with `table-ids`."+ [table-ids :- [:sequential ::lib.schema.id/table]]+ (t2/select [:model/Table :id :db_id :name :display_name] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))++(mu/defn model-less-query-action-ids+ "The ids of the unarchived query Actions without a model outside `excluded-collection-ids`, in name then id+ order."+ [excluded-collection-ids :- [:set ::lib.schema.id/collection]]+ (t2/select-pks-vec :model/Action+ {:where [:and+ [:= :model_id nil]+ [:= :type "query"]+ [:= :archived false]+ (when (seq excluded-collection-ids)+ [:or [:= :collection_id nil] [:not-in :collection_id excluded-collection-ids]])]+ :order-by [[:name :asc] [:id :asc]]}))++(mu/defn field-table-id+ "The Table id of the Field with `field-id`, or nil."+ [field-id :- ::lib.schema.id/field]+ (t2/select-one-fn :table_id :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))++(mu/defn active-tables+ "The active Tables with `table-ids`, in name then id order."+ [table-ids :- [:set ::lib.schema.id/table]]+ (if (seq table-ids)+ (t2/select :model/Table+ {:from [(warehouse-schema-overlay/table-query)]+ :where [:and [:= :active true] [:in :id table-ids]]+ :order-by [[:name :asc] [:id :asc]]})+ []))++(mu/defn published-tables-in-collections+ "The active, published Tables in `collection-ids`, in name then id order."+ [collection-ids :- [:set ::lib.schema.id/collection]]+ (if (seq collection-ids)+ (t2/select :model/Table+ {:from [(warehouse-schema-overlay/table-query)]+ :where [:and+ [:= :active true]+ [:= :is_published true]+ [:in :collection_id collection-ids]]+ :order-by [[:name :asc] [:id :asc]]})+ []))++(mu/defn measure-definition+ "The definition of the Measure with `measure-id`, or nil."+ [measure-id :- ::lib.schema.id/measure]+ (t2/select-one-fn :definition :model/Measure :id measure-id))++(mu/defn measure-definitions+ "The id and definition of the Measures with `measure-ids`."+ [measure-ids :- [:sequential ::lib.schema.id/measure]]+ (t2/select [:model/Measure :id :definition] :id [:in measure-ids]))++(mu/defn collections-with-entity-ids+ "The Collections with `entity-ids`."+ [entity-ids :- [:set :string]]+ (t2/select :model/Collection :entity_id [:in entity-ids]))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate.cljnew file mode 100644index 00000000000..aad3e31ba84--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate.clj@@ -0,0 +1,24 @@+(ns metabase-enterprise.data-apps.generate+ "What a data app's author writes into the app's repository, generated by Metabase: a new app's files, the schema its+ definitions are written against, and the files of its collection those definitions build."+ (:require+ [metabase-enterprise.data-apps.generate.app :as generate.app]+ [metabase-enterprise.data-apps.generate.resources :as generate.resources]+ [metabase-enterprise.data-apps.generate.schemas :as generate.schemas]))++(set! *warn-on-reflection* true)++(defn app+ "A new data app's `data_app.yaml` and collection file, from its `request`."+ [request]+ (generate.app/generate request))++(defn schemas+ "The data app schema as a TypeScript module."+ []+ (generate.schemas/generate))++(defn resources+ "The files of a data app's collection that its `queries` and `actions` definitions build."+ [definitions]+ (generate.resources/generate definitions))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/app.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/app.cljnew file mode 100644index 00000000000..181d5c8dc74--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/app.clj@@ -0,0 +1,57 @@+(ns metabase-enterprise.data-apps.generate.app+ "A new data app's files: its `data_app.yaml` and its collection's YAML, each at the path a remote-sync export+ writes it."+ (:require+ [clojure.string :as str]+ [medley.core :as m]+ [metabase-enterprise.data-apps.config :as data-app.config]+ [metabase-enterprise.data-apps.schema :as data-apps.schema]+ [metabase-enterprise.serialization.core :as serialization]+ [metabase.util :as u]+ [metabase.util.i18n :refer [tru]]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]))++(set! *warn-on-reflection* true)++(def ^:private bundle-path+ "Where the app template builds its bundle, relative to the app's directory."+ "./dist/index.js")++(defn- name->slug+ "The slug `app-name` gives an app: its letters and numbers, lowercased, separated by single dashes."+ [app-name]+ (-> (u/lower-case-en app-name)+ (str/replace #"[^a-z0-9]+" "-")+ (str/replace #"^-+|-+$" "")))++(defn- file+ "The `entity`'s YAML file at the path a remote-sync export writes it, resolving collection paths with `collections`."+ [entity collections]+ {:path (serialization/entity-file-path {:collections collections :unique-name-fns (atom {})} entity)+ :yaml (serialization/entity-yaml entity)})++(mu/defn generate :- ::data-apps.schema/app-files+ "The `data_app.yaml` of a new app named `name`, served at `slug` (from the name by default), and its collection's+ file, both with new entity IDs."+ [{app-name :name :keys [slug description]} :- ::data-apps.schema/app-request]+ (let [slug (or slug (name->slug app-name))+ _ (when-not (mr/validate ::data-apps.schema/slug slug)+ (throw (ex-info (tru "Pass a slug: \"{0}\" is not a valid one." slug) {:status-code 400})))+ app-id (u/generate-nano-id)+ collection-id (u/generate-nano-id)+ collection {:name (str "Data App: " slug)+ :namespace "data-apps"+ :entity_id collection-id+ :serdes/meta [{:model "Collection" :id collection-id+ :label (serialization/slugify-name (str "Data App: " slug))}]}+ app (m/assoc-some {:version data-app.config/supported-app-version+ :name app-name+ :slug slug+ :path bundle-path+ :collection collection-id+ :entity_id app-id+ :serdes/meta [{:model "DataApp" :id app-id :label slug}]}+ :description description)]+ {:files [(file app nil)+ (file collection {collection-id [{:label (:name collection) :key collection-id}]})]}))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/resources.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/resources.cljnew file mode 100644index 00000000000..a41b7ed1da9--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/resources.clj@@ -0,0 +1,238 @@+(ns metabase-enterprise.data-apps.generate.resources+ "The files of a data app's collection: the saved question each `defineQuery` definition builds, and the copies of the+ actions the app runs and of the metrics its queries aggregate, each named and written as a remote-sync export writes+ it in the collection's folder.++ The endpoint is for superusers, who write an app's repository, so nothing here is checked against the caller.+ Opening it to anyone else needs that put back: every source, and every table a query reads at any depth. What a+ routing destination backs is left out, as the typed schema leaves it out."+ (:require+ [clojure.string :as str]+ [clojure.walk :as walk]+ [metabase-enterprise.data-apps.db :as data-apps.db]+ [metabase-enterprise.data-apps.query-definition :as query-definition]+ [metabase-enterprise.data-apps.schema :as data-apps.schema]+ [metabase-enterprise.serialization.core :as serialization]+ [metabase.actions.core :as actions]+ [metabase.api.common :as api]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema]+ [metabase.models.serialization :as serdes]+ [metabase.util.i18n :refer [tru]]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms])+ (:import+ (java.nio.charset StandardCharsets)+ (java.security MessageDigest)+ (java.util Base64)))++(set! *warn-on-reflection* true)++(defn- fail+ "Throws the refusal of an item: a reason the author has to act on, which is not a failure of the serialization."+ [message]+ (throw (ex-info message {::refusal true})))++(defn- with-item-error+ "Calls `thunk`, or answers the `:error` that stopped it, so one item that can't be serialized doesn't fail the+ rest. Anything other than a refusal is logged, so a bug in the serialization leaves a trace on the server."+ [item thunk]+ (try+ (thunk)+ (catch Exception e+ (when-not (::refusal (ex-data e))+ (log/warn e "Could not serialize a data app resource" item))+ {:error (or (ex-message e) (tru "Could not serialize it."))})))++(defn- extract-by-entity-id+ "The entities of `model-name` with `ids`, as serialization extracts them, keyed by entity ID: one extraction for+ them all. Serialization leaves out what it can't extract, and goes on past one it fails on, so an entity missing+ from the result couldn't be extracted; [[extraction-error]] says why."+ [model-name ids]+ (if (seq ids)+ (into {}+ (comp (filter map?)+ (map (juxt (comp :id last :serdes/meta) identity)))+ (serdes/extract-all model-name {:filter-column :id :filter-ids (vec ids) :continue-on-error true}))+ {}))++(defn- extraction-error+ "Why serialization can't extract the `model-name` entity with `id`, or nil when it leaves the entity out without an+ error: the one entity is extracted on its own, with the error let through. The reason is the innermost cause."+ [model-name id]+ (try+ (run! identity (serdes/extract-all model-name {:filter-column :id :filter-ids [id]}))+ nil+ (catch Exception e+ (ex-message (last (take-while some? (iterate ex-cause e)))))))++(defn- cards-read+ "The IDs, as a comma-separated string, of the cards the `model-name` entity `source` references everywhere+ serialization looks, or nil when it references none. A data app's resources can't hold them, so a copy+ referencing one would fail to load."+ [model-name {:keys [parameters] :as source}]+ (some->> (concat (serdes/serialization-dependencies model-name source)+ (serdes/parameters-deps true parameters))+ (keep (fn [path]+ (let [{:keys [model id]} (last path)]+ (when (= "Card" model)+ id))))+ distinct+ not-empty+ sort+ (str/join ", ")))++(defn- check-copyable+ "Throws when the `model-name` entity `source`, labelled `label` and extracted as `serialized`, can't be copied into a+ data app's resources: it's archived, a routing destination backs it, it references a card, or serialization+ couldn't extract it."+ [label model-name {:keys [archived database_id] :as source} serialized]+ (when archived+ (fail (tru "{0} is archived." label)))+ (when (seq (data-apps.db/destination-database-ids #{database_id}))+ (fail (tru "{0} is backed by a routing destination database." label)))+ (when-let [card-ids (cards-read model-name source)]+ (fail (tru "{0} reads card {1}, which a data app''s resources can''t hold." label card-ids)))+ (when-not serialized+ (fail (if-let [cause (extraction-error model-name (:id source))]+ (tru "Could not serialize {0}: {1}" label cause)+ (tru "Could not serialize {0}." label)))))++(mu/defn- build-query :- ::lib.schema/query+ "The query Lib builds from `query-definition`, once its source table and every table it reads at any depth exist."+ [{[{{table-id :id} :source}] :stages, :as query-definition} :- ::query-definition/query-definition]+ (let [table (data-apps.db/table table-id)]+ (when-not table+ (fail (tru "Table {0} does not exist." (str table-id))))+ (let [query (lib/test-query (lib-be/application-database-metadata-provider (:db_id table)) query-definition)]+ (when-not (mr/validate ::lib.schema/query query)+ (fail (tru "The definition does not build a valid query.")))+ (doseq [read-id (sort (into (:table (lib/all-referenced-entity-ids-recursive query))+ (keep :table-id)+ (lib/returned-columns query -1 query {:include-remaps? true})))]+ (when-not (data-apps.db/table read-id)+ (fail (tru "Table {0} does not exist." (str read-id)))))+ query)))++(mu/defn- build-card :- :map+ "The unsaved question named `card-name` with entity ID `entity-id`, holding `query`."+ [card-name :- ms/NonBlankString+ entity-id :- ms/NanoIdString+ query :- ::lib.schema/query]+ {:entity_id entity-id+ :name card-name+ :type :question+ :display :table+ :creator_id api/*current-user-id*+ :database_id (:database query)+ :dataset_query (lib/prepare-for-serialization query)+ :visualization_settings {}+ :parameters []+ :parameter_mappings []})++(def ^:private public-keys+ "What makes a source public or embedded. A copy never is, and the pull refuses a file that says it is."+ [:public_uuid :made_public_by_id :enable_embedding :embedding_params :embedding_type])++(defn- as-copy+ "The extracted `entity` as its copy with `entity-id` in the collection with `collection-id`."+ [entity entity-id collection-id]+ (-> (apply dissoc entity public-keys)+ (assoc :entity_id entity-id :collection_id collection-id)+ (update :serdes/meta #(assoc-in % [(dec (count %)) :id] entity-id))))++(defn- copy-entity-id+ "The entity ID of the copy, in the collection with `collection-id`, of the entity with `source-entity-id`: the same+ every time, so the copy's file is the same every time."+ [collection-id source-entity-id]+ (let [digest (.digest (MessageDigest/getInstance "SHA-256")+ (.getBytes (str collection-id "/" source-entity-id) StandardCharsets/UTF_8))]+ (subs (.encodeToString (.withoutPadding (Base64/getUrlEncoder)) digest) 0 21)))++(defn- metric-copies+ "The metric copies the built `queries` aggregate, as `{[collection-id card-id] {:file .. :entity-id ..}}`."+ [queries]+ (let [uses (into (sorted-set)+ (comp (remove :error)+ (mapcat (fn [{:keys [collection_id built]}]+ (map #(vector collection_id %) (lib/all-source-card-ids built)))))+ queries)+ cards (into {} (map (juxt :id identity)) (data-apps.db/cards-by-ids (into #{} (map second) uses)))+ serialized (extract-by-entity-id "Card" (keys cards))]+ (into (sorted-map)+ (for [[collection-id card-id :as use] uses+ :let [card (cards card-id)]]+ [use {:card card+ :entity-id (some->> card :entity_id (copy-entity-id collection-id))+ :copy (with-item-error+ {:metric card-id}+ (fn []+ (let [label (tru "Metric {0}" (str card-id))]+ (when-not (and card (= :metric (keyword (:type card))))+ (fail (tru "{0} does not exist." label)))+ (check-copyable label "Card" card (serialized (:entity_id card)))+ (as-copy (serialized (:entity_id card))+ (copy-entity-id collection-id (:entity_id card))+ collection-id))))}]))))++(defn- read-metric-copies+ "The extracted question `entity` reading the copies of the metrics it aggregates, in its collection."+ [entity collection-id copies]+ (let [copy-ids (into {}+ (keep (fn [[[copy-collection-id _] {:keys [card entity-id copy]}]]+ (when (and (= copy-collection-id collection-id) (not (:error copy)))+ [(:entity_id card) entity-id])))+ copies)]+ (walk/postwalk #(get copy-ids % %) entity)))++(mu/defn generate :- [:map {:closed true}+ [:queries [:sequential ::data-apps.schema/file]]+ [:actions [:sequential ::data-apps.schema/file]]+ [:metrics [:sequential ::data-apps.schema/file]]]+ "The files of a data app's collection: the saved question of each of `queries`, the copy of each of `actions`, and+ the copies of the metrics the queries aggregate, each named as serialization names it in the collection's folder.+ Each item comes back on its own, with its file or the error that stops it."+ [{:keys [queries actions]} :- [:map {:closed true}+ [:queries {:optional true} [:maybe [:sequential ::data-apps.schema/query]]]+ [:actions {:optional true} [:maybe [:sequential ::data-apps.schema/action]]]]]+ (serdes/with-cache+ (let [unique-name (lib/non-truncating-unique-name-generator)+ ->file (fn [entity]+ {:file (str (unique-name (:entity_id entity) (serialization/slugify-name (:name entity))) ".yaml")+ :yaml (serialization/entity-yaml entity)})+ built (mapv (fn [{:keys [query] :as item}]+ (merge item (with-item-error {:query (:entity_id item)} #(hash-map :built (build-query query)))))+ queries)+ copies (metric-copies built)+ actions-by-id (into {} (map (juxt :id identity))+ (actions/select-actions-for-ids nil (mapv :action_id actions)))+ serialized (extract-by-entity-id "Action" (keys actions-by-id))]+ {:queries (mapv (fn [{:keys [name entity_id collection_id built error]}]+ (if error+ {:error error}+ (with-item-error {:query entity_id}+ (fn []+ (-> (serdes/extract-one "Card" {} (build-card name entity_id built))+ (assoc :collection_id collection_id)+ (read-metric-copies collection_id copies)+ ->file)))))+ built)+ :actions (mapv (fn [{:keys [action_id entity_id collection_id] :as item}]+ (with-item-error item+ (fn []+ (let [label (tru "Action {0}" (str action_id))+ action (actions-by-id action_id)]+ (when-not action+ (fail (tru "{0} does not exist." label)))+ (when (:model_id action)+ (fail (tru "{0} belongs to a model. A data app runs query actions that belong to no model."+ label)))+ (check-copyable label "Action" action (serialized (:entity_id action)))+ (->file (as-copy (serialized (:entity_id action)) entity_id collection_id))))))+ actions)+ :metrics (mapv (fn [{:keys [copy]}]+ (if (:error copy) copy (->file copy)))+ (vals copies))})))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas.cljnew file mode 100644index 00000000000..e06b497ac4d--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas.clj@@ -0,0 +1,70 @@+(ns metabase-enterprise.data-apps.generate.schemas+ "A data app's schema: the TypeScript module describing the root libraries' tables and metrics and the query actions+ that belong to no model, for the `Lib.createTestQuery` DSL; each entry's `/* metadata: {...} */` block carries+ context for humans and agents.++ (-> (fetch-items) create-schema render-typescript)++ [[fetch-items]] is the only impure stage, and reads through+ [[metabase-enterprise.data-apps.generate.schemas.source/SchemaSource]]. Which keys render as runtime data vs+ metadata is policy in `metabase-enterprise.data-apps.generate.schemas.render`; TypeScript syntax lives only in the+ `metabase-enterprise.data-apps.generate.schemas.javascript` printer. The table and metric details lookups+ read-check the current user, so callers outside a request must bind a current-user context first."+ (:require+ [clojure.set :as set]+ [metabase-enterprise.data-apps.generate.schemas.common :as schemas.common]+ [metabase-enterprise.data-apps.generate.schemas.javascript :as schemas.javascript]+ [metabase-enterprise.data-apps.generate.schemas.render :as schemas.render]+ [metabase-enterprise.data-apps.generate.schemas.source :as schemas.source]+ [metabase.system.core :as system])+ (:import+ (java.time Instant)))++(set! *warn-on-reflection* true)++(def Items+ "Fetched schema entities, ready for pure assembly by [[create-schema]]."+ [:map {:closed true}+ [:actions [:sequential :map]]+ [:tables [:sequential :map]]+ [:metrics [:sequential :map]]])++(defn fetch-items+ "Fetches the library's metrics, its published tables and the tables those metrics map, and the query actions that+ belong to no model."+ ([]+ (fetch-items schemas.source/app-db-source))+ ([source]+ (let [{:keys [data-collection-ids metric-collection-ids]} (schemas.source/library-scope source)+ metrics (schemas.source/metrics source metric-collection-ids)+ table-ids (set/union (into #{} (map :id) (schemas.source/library-tables source data-collection-ids))+ (into #{} (mapcat :mappedTableIds) metrics))]+ {:actions (vec (schemas.source/actions source))+ :tables (vec (schemas.source/tables source table-ids))+ :metrics (vec metrics)})))++(defn create-schema+ "Assembles fetched [[Items]] into the schema value; `info` optionally pins `:generated-at` and `:instance-url`,+ which default to the current time and the configured site URL."+ ([items]+ (create-schema items nil))+ ([{:keys [actions tables metrics]} {:keys [generated-at instance-url]}]+ (array-map+ :schemaVersion 2+ :generatedAt (str (or generated-at (Instant/now)))+ :metabase {:instanceUrl (or instance-url (system/site-url))}+ :actions (schemas.common/keyed-map actions)+ :tables (schemas.common/keyed-map tables)+ :metrics (schemas.common/keyed-map metrics))))++(defn render-typescript+ "Renders a schema value as an ES module of `as const` TypeScript constants."+ [schema]+ (-> schema+ schemas.render/schema->ast+ schemas.javascript/render-js))++(defn generate+ "The data app's schema as a TypeScript module."+ []+ (-> (fetch-items) create-schema render-typescript))diff --git a/src/metabase/typed_schemas/schema/action.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/action.cljsimilarity index 82%rename from src/metabase/typed_schemas/schema/action.cljrename to enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/action.cljindex dc8f403866c..23881a7e315 100644--- a/src/metabase/typed_schemas/schema/action.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/action.clj@@ -1,14 +1,13 @@-(ns metabase.typed-schemas.schema.action- "Typed schema generation for actions that belong to no model."+(ns metabase-enterprise.data-apps.generate.schemas.action+ "The schema of the query actions that belong to no model."(:require[medley.core :as m]+ [metabase-enterprise.data-apps.db :as data-apps.db]+ [metabase-enterprise.data-apps.generate.schemas.common :as schemas.common][metabase.actions.core :as actions][metabase.lib.core :as lib][metabase.lib.schema.common :as lib.schema.common][metabase.permissions.core :as perms]- [metabase.typed-schemas.common :as common]- [metabase.typed-schemas.db :as typed-schemas.db]- [metabase.typed-schemas.schema.common :as schema.common][metabase.util :as u]))(set! *warn-on-reflection* true)@@ -68,7 +67,7 @@hidden? (hidden-parameter-ids action)](m/assoc-some{:kind "action"- :key (common/generated-key name id)+ :key (schemas.common/generated-key name id):id id:name name:type "query"@@ -79,13 +78,13 @@:entityId entity_id)))(defn action-schemas- "Returns schema entries for the query actions without a model among `database-ids` (nil for unscoped),- leaving out the copies data apps own."- [database-ids]- (let [ids (typed-schemas.db/model-less-query-action-ids database-ids (set (perms/data-app-collection-ids)))+ "Returns schema entries for the query actions without a model, leaving out the copies data apps own and those a+ routing destination backs."+ []+ (let [ids (data-apps.db/model-less-query-action-ids (set (perms/data-app-collection-ids)))details-by-id (when (seq ids) (u/index-by :id (actions/select-actions-for-ids nil ids)))details (keep details-by-id ids)- destination-ids (schema.common/destination-db-ids (into #{} (keep :database_id) details))]+ destination-ids (data-apps.db/destination-database-ids (into #{} (keep :database_id) details))](into [](comp (remove #(contains? destination-ids (:database_id %)))(map action-schema))diff --git a/src/metabase/typed_schemas/common.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/common.cljsimilarity index 67%rename from src/metabase/typed_schemas/common.cljrename to enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/common.cljindex 6c3297258fc..fbef3e09c27 100644--- a/src/metabase/typed_schemas/common.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/common.clj@@ -1,8 +1,14 @@-(ns metabase.typed-schemas.common- "Shared helpers for typed-schema generation."+(ns metabase-enterprise.data-apps.generate.schemas.common+ "Shared helpers for generating a data app's schema."(:require[clojure.string :as str][medley.core :as m]+ [metabase-enterprise.data-apps.db :as data-apps.db]+ [metabase.audit-app.core :as audit]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.metabot.core :as metabot]+ [metabase.premium-features.core :as premium-features][metabase.types.core][metabase.util :as u]))@@ -117,3 +123,37 @@(dissoc :keyDisambiguator)(assoc :key key))])entities ks))))++(defn without-unavailable-cards+ "`cards` without any backed by a routing destination database, and, while the audit feature is off, without the+ audit database's and the audit collection's: the card details lookup refuses one in the audit collection, and one+ on the audit database would be listed without its table."+ [cards]+ (let [destination-ids (data-apps.db/destination-database-ids (into #{} (keep :database_id) cards))+ audit-off? (not (premium-features/enable-audit-app?))]+ (cond->> cards+ (seq destination-ids) (remove #(contains? destination-ids (:database_id %)))+ audit-off? (remove #(or (= audit/audit-db-id (:database_id %))+ (some-> (:collection_id %) audit/is-collection-id-audit?))))))++(defn aggregation-result-column-with-metadata-provider+ "Returns an aggregation result column using an existing metadata provider."+ [metadata-provider query-definition]+ (try+ (let [query (lib/query metadata-provider query-definition)+ aggregation-column (m/find-first #(= (:lib/source %) :source/aggregations)+ (lib/returned-columns query))]+ (when aggregation-column+ (metabot/->result-column query aggregation-column)))+ (catch Exception _+ nil)))++(defn aggregation-result-column+ "Returns the first aggregation result column for a saved query definition, for metrics and measures."+ [database-id query-definition]+ (try+ (aggregation-result-column-with-metadata-provider+ (lib-be/application-database-metadata-provider database-id)+ query-definition)+ (catch Exception _+ nil)))diff --git a/src/metabase/typed_schemas/javascript.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/javascript.cljsimilarity index 98%rename from src/metabase/typed_schemas/javascript.cljrename to enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/javascript.cljindex 2f5f21a3c04..66ba8a3e37f 100644--- a/src/metabase/typed_schemas/javascript.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/javascript.clj@@ -1,4 +1,4 @@-(ns metabase.typed-schemas.javascript+(ns metabase-enterprise.data-apps.generate.schemas.javascript"A small TypeScript AST encoded as tagged vectors, and a printer for it.The AST covers exactly the subset of TypeScript that typed-schema modules@@ -53,7 +53,7 @@The printer is deliberately option-free and policy-free: decisions about*what* to emit (runtime keys, metadata, compaction) belong in- `metabase.typed-schemas.render`, which builds this AST. If the output needs+ `metabase-enterprise.data-apps.generate.schemas.render`, which builds this AST. If the output needsnew syntax, add a node type and its printer here rather than concatenatingTypeScript strings elsewhere."(:requirediff --git a/src/metabase/typed_schemas/schema/metric.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/metric.cljsimilarity index 89%rename from src/metabase/typed_schemas/schema/metric.cljrename to enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/metric.cljindex 9150b8cdb53..2643b896537 100644--- a/src/metabase/typed_schemas/schema/metric.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/metric.clj@@ -1,15 +1,14 @@-(ns metabase.typed-schemas.schema.metric- "Typed schema generation for metrics and metric dimensions."+(ns metabase-enterprise.data-apps.generate.schemas.metric+ "The schema of metrics, with their dimensions."(:require[medley.core :as m]+ [metabase-enterprise.data-apps.db :as data-apps.db]+ [metabase-enterprise.data-apps.generate.schemas.common :as schemas.common]+ [metabase-enterprise.data-apps.generate.schemas.table :as schemas.table][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.metabot.core :as metabot][metabase.metrics.core :as metrics]- [metabase.typed-schemas.common :as common]- [metabase.typed-schemas.db :as typed-schemas.db]- [metabase.typed-schemas.schema.common :as schema.common]- [metabase.typed-schemas.schema.table :as schema.table][metabase.util :as u][metabase.util.log :as log]))@@ -18,7 +17,7 @@(defn- metric-result-column"Returns the metric aggregation result column inferred by Lib."[card]- (schema.common/aggregation-result-column (:database_id card) (:dataset_query card)))+ (schemas.common/aggregation-result-column (:database_id card) (:dataset_query card)))(defn- metric-filters"Returns the display names of the filters `card`'s query applies. The schema exposes a metric only as an@@ -68,7 +67,7 @@(defn- persisted-dimension->column"Converts persisted metric dimensions to column-shaped maps.- Persisted metric dimensions use kebab-case field/type keys, while [[common/column-schema]] expects+ Persisted metric dimensions use kebab-case field/type keys, while [[schemas.common/column-schema]] expectsresult-column-style snake_case keys."[dimension](let [field-id (some-> dimension :sources first :field-id)]@@ -93,7 +92,7 @@(:table-id dimension)(if field-table-ids(get field-table-ids (dimension-field-id dimension))- (schema.table/table-by-field-id (dimension-field-id dimension))))))+ (schemas.table/table-by-field-id (dimension-field-id dimension))))))(defn- field-table-ids"Returns backing table ids for dimensions that need field-based resolution."@@ -105,7 +104,7 @@(when (seq field-ids)(into {}(map (juxt :id :table_id))- (typed-schemas.db/field-ids-and-table-ids field-ids)))))+ (data-apps.db/field-ids-and-table-ids field-ids)))))(defn- dimension-schema"Returns the schema for a metric dimension."@@ -122,9 +121,9 @@(persisted-dimension->column dimension)dimension)](m/assoc-some- (assoc (common/column-schema column)+ (assoc (schemas.common/column-schema column):type "column"- :key (common/generated-key (:name column) dimension-id)+ :key (schemas.common/generated-key (:name column) dimension-id):id (str dimension-id)):sourceName (when (integer? table-id)(get table-source-name-by-id table-id))@@ -160,14 +159,14 @@"Syncs and returns persisted metric dimensions with mapping metadata."[{:keys [id]}](metrics/sync-dimensions! :metadata/metric id)- (let [{:keys [dimensions dimension_mappings]} (typed-schemas.db/card-dimensions id)]+ (let [{:keys [dimensions dimension_mappings]} (data-apps.db/card-dimensions id)](enrich-dimensions-with-mappings dimensions dimension_mappings)))(defn- table-source-rows"Returns table rows for table-backed metric dimensions."[table-ids](when (seq table-ids)- (typed-schemas.db/table-names table-ids)))+ (data-apps.db/table-names table-ids)))(defn- table-key-disambiguators"Returns table display keys used to disambiguate compacted metric dimensions."@@ -175,7 +174,7 @@(when (seq table-rows)(->> table-rows(map (fn [{:keys [id name display_name]}]- [id (common/pascal-case (common/generated-key (or display_name name) id))]))+ [id (schemas.common/pascal-case (schemas.common/generated-key (or display_name name) id))]))(into {}))))(defn- table-source-names@@ -261,13 +260,13 @@"Returns the schema for a metric and its queryable dimensions."[{:keys [id name description verified portable_entity_id base_table_portable_fk] :as details}card]- (let [result-column (or (some-> (metric-result-column card) common/column-schema)+ (let [result-column (or (some-> (metric-result-column card) schemas.common/column-schema)(fallback-metric-column details))source-card-id-value (source-card-id card){:keys [dimension-schemas mapped-table-ids]} (metric-dimension-schemas id details source-card-id-value)](m/assoc-some{:type "metric"- :key (common/generated-key name id)+ :key (schemas.common/generated-key name id):id id:name name:columns [result-column]}@@ -280,7 +279,7 @@:verified (when verified true):sourceTable (source-table-schema base_table_portable_fk):mappedTableIds (not-empty mapped-table-ids)- :dimensions (not-empty (common/keyed-map dimension-schemas)))))+ :dimensions (not-empty (schemas.common/keyed-map dimension-schemas)))))(defn- references-saved-card?"Whether `metric`'s query depends on a saved card anywhere, not only as its stage-0 source.@@ -294,10 +293,10 @@(boolean (seq (lib/all-source-card-ids query))))))(defn metric-schemas- "Returns metric schemas, with optional database and collection scopes."- [database-ids collection-ids]+ "Returns the schemas of the metrics in the collections with `collection-ids`."+ [collection-ids](for [metric (remove source-card-id- (schema.common/select-schema-cards :metric database-ids collection-ids))+ (schemas.common/without-unavailable-cards (data-apps.db/metric-cards-in-collections collection-ids)));; A data app can copy only metrics that resolve entirely from tables: a copy reading a;; saved question references a card outside the app's resources, which the pull refuses.;; `source-card-id` sees only stage 0, so a table-sourced metric joining a saved questiondiff --git a/src/metabase/typed_schemas/render.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/render.cljsimilarity index 95%rename from src/metabase/typed_schemas/render.cljrename to enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/render.cljindex 27aae76e987..176ff810f1c 100644--- a/src/metabase/typed_schemas/render.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/render.clj@@ -1,12 +1,12 @@-(ns metabase.typed-schemas.render+(ns metabase-enterprise.data-apps.generate.schemas.render"Converts semantic schema values into the TypeScript AST.This is the pure policy stage of the typed-schema pipeline: it decides whichschema keys become runtime data, which become `/* metadata */` blocks, and howmetric dimensions compact into `pickFields(...)` calls. The output is the- tagged-vector AST described in [[metabase.typed-schemas.javascript]];+ tagged-vector AST described in [[metabase-enterprise.data-apps.generate.schemas.javascript]];[[schema->ast]] produces it, and the public- [[metabase.typed-schemas.core/render-typescript]] composes it with the+ [[metabase-enterprise.data-apps.generate.schemas/render-typescript]] composes it with theprinter.Nothing here touches the database or builds strings: new rendering behavior@@ -15,7 +15,7 @@structurally in tests."(:require[clojure.string :as str]- [metabase.typed-schemas.javascript :as javascript]))+ [metabase-enterprise.data-apps.generate.schemas.javascript :as schemas.javascript]))(set! *warn-on-reflection* true)@@ -25,9 +25,7 @@;; entity for coding agents and humans, preserving useful context without;; making the runtime object larger or more ambiguous than the DSL needs.(def ^:private schema-render-policy- {:question {:runtime [:type :id :name :display :columns :parameters]- :metadata [:entityId :description :verified]}- :table {:runtime [:type :id :name :fields :segments :measures]+ {:table {:runtime [:type :id :name :fields :segments :measures]:metadata [:entityId :description :databaseName :schemaName :tableName]}:field {:runtime [:type :name :sourceName :jsType :fieldId :tableId:baseType :effectiveType :defaultTemporalBucket]@@ -50,8 +48,7 @@;; special-cased in [[entity->node]] because compaction mixes call expressions;; with nested dimension maps.(def ^:private entity-children- {:question {:columns :column}- :table {:fields :field, :segments :segment, :measures :measure}+ {:table {:fields :field, :segments :segment, :measures :measure}:measure {:columns :column}:metric {:columns :column}})@@ -98,7 +95,7 @@metadata) emit every key and no metadata block."[value](cond- (javascript/expression? value) value+ (schemas.javascript/expression? value) value(map? value) (into [:obj] (for [[entry-key entry-value] value][entry-key (generic->node entry-value)]))(sequential? value) (into [:arr] (map generic->node) value)@@ -132,7 +129,7 @@[dimensions](into [:obj](for [[entry-key group] dimensions]- [entry-key (if (javascript/expression? group)+ [entry-key (if (schemas.javascript/expression? group)group(keyed-entities->obj group :metric-dimension))])))@@ -321,17 +318,16 @@[schema](boolean(some (fn [metric]- (some javascript/call? (vals (:dimensions metric))))+ (some schemas.javascript/call? (vals (:dimensions metric))))(vals (:metrics schema)))))(def ^:private top-level-keys- [:questions :actions :tables :metrics])+ [:actions :tables :metrics])(defn- section->node"Converts one top-level schema section into an object expression."[section-key section](case section-key- :questions (keyed-entities->obj section :question):actions (generic->node section):tables (keyed-entities->obj section :table):metrics (keyed-entities->obj section :metric)))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/source.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/source.cljnew file mode 100644index 00000000000..b496aad6737--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/source.clj@@ -0,0 +1,65 @@+(ns metabase-enterprise.data-apps.generate.schemas.source+ "Data access for a data app's schema, reified as a protocol.++ [[SchemaSource]] names every read that [[metabase-enterprise.data-apps.generate.schemas/fetch-items]] performs, so+ the data-access surface is one definition. [[app-db-source]] is the production implementation; tests reify the+ protocol with literal values. All methods return the shaped schema entities of the+ `metabase-enterprise.data-apps.generate.schemas.*` builders, except [[library-tables]], which returns raw table rows+ (only `:id` is consumed)."+ (:require+ [metabase-enterprise.data-apps.db :as data-apps.db]+ [metabase-enterprise.data-apps.generate.schemas.action :as schemas.action]+ [metabase-enterprise.data-apps.generate.schemas.metric :as schemas.metric]+ [metabase-enterprise.data-apps.generate.schemas.table :as schemas.table]+ [metabase.collections.models.collection :as collection]))++(set! *warn-on-reflection* true)++(def ^:private library-root-entity-ids+ "Entity ids of the root data library and root metrics library collections."+ #{"librarylibrarydatadat" "librarylibrarymetrics"})++(def LibraryScope+ "The library collection tree, classified by collection type: `:data-collection-ids` hold the published tables,+ `:metric-collection-ids` the metrics."+ [:map {:closed true}+ [:data-collection-ids [:set :int]]+ [:metric-collection-ids [:set :int]]])++(defn- app-db-library-scope+ "The [[LibraryScope]] of the root libraries and their descendants, empty where the instance has no library."+ []+ (let [roots (filter #(contains? collection/library-collection-types (:type %))+ (data-apps.db/collections-with-entity-ids library-root-entity-ids))+ collections (concat roots (when (seq roots) (collection/descendants-flat-for roots)))+ ids-of-type (fn [collection-type]+ (into #{} (comp (filter #(= (:type %) collection-type)) (map :id)) collections))]+ {:data-collection-ids (ids-of-type collection/library-data-collection-type)+ :metric-collection-ids (ids-of-type collection/library-metrics-collection-type)}))++(defprotocol SchemaSource+ "The data-access patterns behind schema generation."+ (library-scope [source]+ "The [[LibraryScope]] of the root libraries.")+ (actions [source]+ "Schema entities for query actions that belong to no model.")+ (metrics [source collection-ids]+ "Schema entities for the metrics in `collection-ids`.")+ (tables [source table-ids]+ "Schema entities for the tables with `table-ids`.")+ (library-tables [source collection-ids]+ "Published table rows in `collection-ids`."))++(def app-db-source+ "The production [[SchemaSource]], backed by the application database."+ (reify SchemaSource+ (library-scope [_]+ (app-db-library-scope))+ (actions [_]+ (schemas.action/action-schemas))+ (metrics [_ collection-ids]+ (vec (schemas.metric/metric-schemas collection-ids)))+ (tables [_ table-ids]+ (vec (schemas.table/table-schemas (schemas.table/select-tables table-ids))))+ (library-tables [_ collection-ids]+ (vec (schemas.table/select-library-tables collection-ids)))))diff --git a/src/metabase/typed_schemas/schema/table.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/table.cljsimilarity index 70%rename from src/metabase/typed_schemas/schema/table.cljrename to enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/table.cljindex bf48fc3e8be..c44967b6626 100644--- a/src/metabase/typed_schemas/schema/table.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/table.clj@@ -1,14 +1,13 @@-(ns metabase.typed-schemas.schema.table- "Typed schema generation for tables, fields, segments and measures."+(ns metabase-enterprise.data-apps.generate.schemas.table+ "The schema of tables, with their fields, segments and measures."(:require[medley.core :as m]+ [metabase-enterprise.data-apps.db :as data-apps.db]+ [metabase-enterprise.data-apps.generate.schemas.common :as schemas.common][metabase.audit-app.core :as audit][metabase.lib-be.core :as lib-be][metabase.metabot.core :as metabot]- [metabase.premium-features.core :as premium-features]- [metabase.typed-schemas.common :as common]- [metabase.typed-schemas.db :as typed-schemas.db]- [metabase.typed-schemas.schema.common :as schema.common]))+ [metabase.premium-features.core :as premium-features]))(set! *warn-on-reflection* true)@@ -16,7 +15,7 @@"Returns the table for a given field id."[field-id](when (integer? field-id)- (typed-schemas.db/field-table-id field-id)))+ (data-apps.db/field-table-id field-id)))(defn field-schema"Returns the schema for a table field."@@ -28,9 +27,9 @@(let [field-id (or id field_id)table-id (or table-id (:table_id field) (:table-id field) (table-by-field-id field-id))](m/assoc-some- (assoc (common/column-schema field)+ (assoc (schemas.common/column-schema field):type "column"- :key (common/generated-key (:name field) field-id)+ :key (schemas.common/generated-key (:name field) field-id):id field-id):sourceName source-name:fieldId (when (integer? field-id) field-id)@@ -38,28 +37,24 @@:defaultTemporalBucket (:unit field)))))(defn- without-unavailable-tables- "`tables` without any backed by a destination (routed) database -- see [[schema.common/destination-db-ids]] -- and- without the audit database's while the audit feature is off, since the table details lookup refuses those."+ "`tables` without any backed by a routing destination database, and without the audit database's while the audit+ feature is off, since the table details lookup refuses those."[tables]- (let [destination-ids (schema.common/destination-db-ids (into #{} (keep :db_id) tables))+ (let [destination-ids (data-apps.db/destination-database-ids (into #{} (keep :db_id) tables))audit-off? (not (premium-features/enable-audit-app?))](cond->> tables(seq destination-ids) (remove #(contains? destination-ids (:db_id %)))audit-off? (remove #(= audit/audit-db-id (:db_id %))))))(defn select-tables- "Returns the active tables, with optional database and table-id scopes.-- Library and database endpoint paths both need the same active-table rules;- only their id filters differ."- [database-ids table-ids]- (->> (typed-schemas.db/active-tables-in-scope database-ids table-ids)- (without-unavailable-tables)))+ "Returns the active tables with `table-ids`."+ [table-ids]+ (without-unavailable-tables (data-apps.db/active-tables table-ids)))(defn select-library-tables- "Returns published tables from the library based on the given scope."- [{:keys [data-collection-ids]}]- (->> (typed-schemas.db/published-library-tables-in-collections data-collection-ids)+ "Returns the published tables in the library collections with `collection-ids`."+ [collection-ids]+ (->> (data-apps.db/published-tables-in-collections collection-ids)(without-unavailable-tables)))(defn segment-schema@@ -67,7 +62,7 @@[table-id {:keys [id name description display-name portable-entity-id portable_entity_id]}](m/assoc-some{:type "segment"- :key (common/generated-key (or display-name name) id)+ :key (schemas.common/generated-key (or display-name name) id):id id:tableId table-id:name name}@@ -89,8 +84,8 @@measure remains usable with a fallback column."[database-id measure-id](try- (when-let [definition (typed-schemas.db/measure-definition measure-id)]- (schema.common/aggregation-result-column database-id definition))+ (when-let [definition (data-apps.db/measure-definition measure-id)]+ (schemas.common/aggregation-result-column database-id definition))(catch Exception _nil)))@@ -101,13 +96,13 @@(when (seq measure-ids)(let [definition-by-measure-id (into {}(map (juxt :id :definition))- (typed-schemas.db/measure-definitions measure-ids))]+ (data-apps.db/measure-definitions measure-ids))](try(let [metadata-provider (lib-be/application-database-metadata-provider database-id)](into {}(keep (fn [{:keys [id]}](when-let [definition (get definition-by-measure-id id)]- [id (schema.common/aggregation-result-column-with-metadata-provider metadata-provider definition)])))+ [id (schemas.common/aggregation-result-column-with-metadata-provider metadata-provider definition)])))measures));; Result-column inference is best effort; callers fall back to an unknown column.(catch Exception _@@ -120,11 +115,11 @@([table-id _database-id result-column {:keys [id name description display-name portable-entity-id portable_entity_id]}](m/assoc-some{:type "measure"- :key (common/generated-key (or display-name name) id)+ :key (schemas.common/generated-key (or display-name name) id):id id:tableId table-id:name name- :columns [(or (some-> result-column common/column-schema)+ :columns [(or (some-> result-column schemas.common/column-schema)(fallback-measure-column {:name (or display-name name)}))]}:entityId (or portable_entity_id portable-entity-id):description description)))@@ -132,20 +127,20 @@(defn table-schema"Returns the schema for a table."[{:keys [id name description display_name database_id database_name database_schema portable_entity_id fields segments measures]}]- (let [segments-map (common/keyed-map (map #(segment-schema id %) segments))+ (let [segments-map (schemas.common/keyed-map (map #(segment-schema id %) segments))measure-result-column-by-id (measure-result-columns database_id measures)- measures-map (common/keyed-map (map #(measure-schema id database_id- (get measure-result-column-by-id (:id %))- %)- measures))]+ measures-map (schemas.common/keyed-map (map #(measure-schema id database_id+ (get measure-result-column-by-id (:id %))+ %)+ measures))](m/assoc-some{:type "table"- :key (common/generated-key (or display_name name) id)+ :key (schemas.common/generated-key (or display_name name) id):id id:name (or display_name name):databaseName database_name:tableName name- :fields (common/keyed-map (map #(field-schema % name id) fields))}+ :fields (schemas.common/keyed-map (map #(field-schema % name id) fields))}:entityId portable_entity_id:description description:schemaName database_schemadiff --git a/enterprise/backend/src/metabase_enterprise/data_apps/resource_serialization.clj b/enterprise/backend/src/metabase_enterprise/data_apps/resource_serialization.cljdeleted file mode 100644index d2b467677d5..00000000000--- a/enterprise/backend/src/metabase_enterprise/data_apps/resource_serialization.clj+++ /dev/null@@ -1,234 +0,0 @@-(ns metabase-enterprise.data-apps.resource-serialization- "What the files of a data app's collection are written from, as serialization writes it: the saved question that holds- the query Metabase builds from each `defineQuery` definition, each action the app runs, and the metrics its queries- aggregate. Nothing here references an entity by numeric ID, so the author copies it into the app's collection as it- is, apart from what makes it a copy.-- The endpoint is for superusers, who write an app's repository, so nothing here is checked against the caller.- Opening it to anyone else needs that put back: every source, and every table a query reads at any depth. What a- routing destination backs is left out, as the typed schema leaves it out."- (:require- [clojure.string :as str]- [clojure.walk :as walk]- [metabase-enterprise.data-apps.db :as data-apps.db]- [metabase-enterprise.data-apps.query-definition :as query-definition]- [metabase-enterprise.serialization.dump :as serialization.dump]- [metabase.actions.core :as actions]- [metabase.api.common :as api]- [metabase.lib-be.core :as lib-be]- [metabase.lib.core :as lib]- [metabase.lib.schema :as lib.schema]- [metabase.models.serialization :as serdes]- [metabase.util :as u]- [metabase.util.i18n :refer [tru]]- [metabase.util.log :as log]- [metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr]- [metabase.util.malli.schema :as ms]))--(set! *warn-on-reflection* true)--(defn- fail- "Throws the refusal of an item: a reason the author has to act on, which is not a failure of the serialization."- [message]- (throw (ex-info message {::refusal true})))--(defn- with-item-error- "Calls `serialize`, or answers `item` with the `:error` that stopped it, so one item that can't be serialized doesn't- fail the rest. Anything other than a refusal is logged, so a bug in the serialization leaves a trace on the server."- [item serialize]- (try- (serialize)- (catch Exception e- (when-not (::refusal (ex-data e))- (log/warn e "Could not serialize a data app resource" item))- (assoc item :error (or (ex-message e) (tru "Could not serialize it."))))))--(defn- extract-by-entity-id- "The entities of `model-name` with `ids`, as serialization extracts them, keyed by entity ID: one extraction for- them all. Serialization leaves out what it can't extract (an exploration's card), and goes on past one it fails- on, so an entity missing from the result couldn't be extracted; [[extraction-error]] says why."- [model-name ids]- (if (seq ids)- (into {}- (comp (filter map?)- (map (juxt (comp :id last :serdes/meta) identity)))- (serdes/extract-all model-name {:filter-column :id :filter-ids (vec ids) :continue-on-error true}))- {}))--(defn- extraction-error- "Why serialization can't extract the `model-name` entity with `id`, or nil when it leaves the entity out without an- error: the one entity is extracted on its own, with the error let through. Serialization wraps what fails inside- an extraction in \"Error extracting ...\" at each level, so the reason is the innermost cause."- [model-name id]- (try- ;; reduced rather than seq'd: the extraction is a reducible, not a seq- (run! identity (serdes/extract-all model-name {:filter-column :id :filter-ids [id]}))- nil- (catch Exception e- (ex-message (last (take-while some? (iterate ex-cause e)))))))--(defn- cards-read- "The IDs, as a comma-separated string, of the cards the `model-name` entity `source` references everywhere- serialization looks (its query, its visualization settings, its parameters' value sources, ...), except- `own-cards`, or nil when it references none. A data app's resources can't hold them, so a copy referencing one- would fail to load."- [model-name {:keys [parameters] :as source} own-cards]- (some->> (concat (serdes/serialization-dependencies model-name source)- ;; An action's dependencies leave out its parameters, which a card's include.- (serdes/parameters-deps true parameters))- (keep (fn [path]- (let [{:keys [model id]} (last path)]- (when (and (= "Card" model) (not (contains? own-cards id)))- id))))- distinct- not-empty- sort- (str/join ", ")))--(defn- check-copyable- "Throws when the `model-name` entity `source`, labelled `label` and extracted as `serialized`, can't be copied into a- data app's resources: it's archived (the pull refuses it), a routing destination backs it, it references a card other- than `own-cards`, or serialization couldn't extract it, with the reason when it gave one."- [label model-name {:keys [archived database_id] :as source} serialized own-cards]- (when archived- (fail (tru "{0} is archived." label)))- (when (seq (data-apps.db/destination-database-ids #{database_id}))- (fail (tru "{0} is backed by a routing destination database." label)))- (when-let [card-ids (cards-read model-name source own-cards)]- (fail (tru "{0} reads card {1}, which a data app''s resources can''t hold." label card-ids)))- (when-not serialized- (fail (if-let [cause (extraction-error model-name (:id source))]- (tru "Could not serialize {0}: {1}" label cause)- (tru "Could not serialize {0}." label)))))--(def ^:private public-keys- "What makes a source public or embedded. A copy never is, and the pull refuses a file that says it is."- [:public_uuid :made_public_by_id :enable_embedding :embedding_params :embedding_type])--(defn- as-written- "`entity` as the author writes it: in the key order serialization writes, without the keys it leaves unset, which- the format omits, and without what makes the source public or embedded. The query is kept whole; nothing in it is- unset."- [entity]- (let [query (:dataset_query entity)]- (serialization.dump/serialization-deep-sort- (cond-> (walk/postwalk (fn [x] (if (map? x) (into {} (remove (comp nil? val)) x) x))- (apply dissoc entity :dataset_query public-keys))- query (assoc :dataset_query query)))))--(defn- export-name->card-name- "The saved question's name from the definition's export name: its words, as `ProductsList` is `Products list`."- [export]- (-> export- (str/replace #"([a-z0-9])([A-Z])" "$1 $2")- (str/replace #"[\s_]+" " ")- u/lower-case-en- str/capitalize))--(defn- question-card- "The saved question an author writes for a `defineQuery` definition, as a card for serialization to extract: the- query Metabase `built`, created by the caller, with the definition's entity ID, which the definition carries so the- file is complete as printed. The app's collection is set on the extracted entity, since serialization resolves a- collection by numeric ID."- [{:keys [export entity_id]} built]- {:entity_id entity_id- :name (export-name->card-name export)- :type :question- :display :table- :creator_id api/*current-user-id*- :database_id (:database built)- :dataset_query (lib/prepare-for-serialization built)- :visualization_settings {}- :parameters []- :parameter_mappings []})--(mu/defn- built-query- "The query Metabase builds from `query-definition`, as the dev preview does, once its source table and every table- it reads at any depth, through a foreign key, a metric it aggregates or a column remapping, exist as the typed- schema lists them. (A routing destination has no tables of its own, only cards.)"- [{[{{table-id :id} :source}] :stages, :as query-definition} :- ::query-definition/query-definition]- (let [table (data-apps.db/table table-id)]- (when-not table- (fail (tru "Table {0} does not exist." (str table-id))))- (let [built (lib/test-query (lib-be/application-database-metadata-provider (:db_id table)) query-definition)]- ;; The request schema accepts what a type lets through, such as a fractional limit, and the checks in lib- ;; that would refuse it are off in production: an invalid query must not serialize and then fail when it runs.- (when-not (mr/validate ::lib.schema/query built)- (fail (tru "The definition does not build a valid query.")))- ;; Types generated before a table was deactivated still reach its columns through a foreign key, as does the- ;; query of a metric the built query holds only the ID of, and a column remapped to display another table's- ;; column joins that table only when the query runs. The query would fail on the table when it runs.- (doseq [read-id (sort (into (:table (lib/all-referenced-entity-ids-recursive built))- (keep :table-id)- (lib/returned-columns built -1 built {:include-remaps? true})))]- (when-not (data-apps.db/table read-id)- (fail (tru "Table {0} does not exist." (str read-id)))))- built)))--(mu/defn- serialize-query- "A `defineQuery` definition as `{:export :entity :metric_ids}`, the entity being the saved question that holds the- query Metabase builds from it, in the app's collection with `collection-entity-id`, or `{:export :error}`."- [collection-entity-id :- ms/NanoIdString- {:keys [export query] :as definition} :- [:map {:closed true}- [:export ms/NonBlankString]- [:entity_id ms/NanoIdString]- [:query ::query-definition/query-definition]]]- (with-item-error- {:export export}- (fn []- (let [built (built-query query)]- {:export export- :entity (as-written (assoc (serdes/extract-one "Card" {} (question-card definition built))- :collection_id collection-entity-id))- :metric_ids (vec (sort (lib/all-source-card-ids built)))}))))--(defn- serialize-metric- "The metric `card` as `{:id :entity}`, or `{:id :error}` when the app can't copy it. `serialized` is what- serialization extracts for it, if anything."- [card-id card serialized]- (with-item-error- {:id card-id}- (fn []- (let [label (tru "Metric {0}" (str card-id))]- (when-not (and card (= :metric (keyword (:type card))))- (fail (tru "{0} does not exist." label)))- (check-copyable label "Card" card serialized #{})- {:id card-id, :entity (as-written serialized)}))))--(defn- serialize-action- "The action with `action-id` as `{:id :entity}`, or `{:id :error}` when the app can't copy it. `serialized` is what- serialization extracts for the action, if anything. A data app runs only actions that belong to no model, as the typed- schema lists only those."- [action-id action serialized]- (with-item-error- {:id action-id}- (fn []- (let [label (tru "Action {0}" (str action-id))]- (when-not action- (fail (tru "{0} does not exist." label)))- (when (:model_id action)- (fail (tru "{0} belongs to a model. A data app runs query actions that belong to no model." label)))- (check-copyable label "Action" action serialized #{})- {:id action-id, :entity (as-written serialized)}))))--(defn serialize-resources- "Serialize the saved question built from each of `queries` (`{:export <name> :query <definition> :entity_id <id>}`),- in the app's collection with `collection-entity-id`, the actions with `action-ids`, and the metrics the queries- aggregate. Each item comes back on its own, with its serialization or the error that stops it, so one item that- can't be serialized doesn't hide the rest. A query lists the entity IDs of the metrics it references, which the- author points at the app's copies."- [collection-entity-id queries action-ids]- (serdes/with-cache- (let [queries (mapv (partial serialize-query collection-entity-id) queries)- actions-by-id (into {} (map (juxt :id identity)) (actions/select-actions-for-ids nil action-ids))- metric-ids (into (sorted-set) (mapcat :metric_ids) queries)- cards-by-id (into {} (map (juxt :id identity)) (data-apps.db/cards-by-ids metric-ids))- serialized-card (comp (extract-by-entity-id "Card" (keys cards-by-id)) :entity_id cards-by-id)- serialized-action (comp (extract-by-entity-id "Action" (keys actions-by-id)) :entity_id actions-by-id)]- {:queries (mapv (fn [{:keys [metric_ids] :as query}]- (cond-> (dissoc query :metric_ids)- (not (:error query)) (assoc :metrics (mapv (comp :entity_id cards-by-id) metric_ids))))- queries)- :actions (mapv #(serialize-action % (actions-by-id %) (serialized-action %)) action-ids)- :metrics (mapv #(serialize-metric % (cards-by-id %) (serialized-card %)) metric-ids)})))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj b/enterprise/backend/src/metabase_enterprise/data_apps/schema.cljindex 5f5bc0138f7..48736427bf3 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj@@ -2,6 +2,7 @@"Malli schemas for the data-apps module."(:require[clojure.string :as str]+ [metabase-enterprise.data-apps.query-definition :as query-definition][metabase.util :as u][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))@@ -12,7 +13,7 @@(def ^:private reserved-slugs"Slugs that collide with literal `/api/apps/*` sub-routes, so an app with one would be unreachable."- #{"repo-status" "sandbox-host"})+ #{"generate" "repo-status" "sandbox-host"})(defn- decoders"Schema properties decoding a value with `f` both in [[metabase.lib.core/normalize]] and at the API boundary."@@ -123,3 +124,41 @@[:name ::slug][:display_name ::display-name][:bundle_path ::bundle-path]]])++(mr/def ::query+ "A data app `defineQuery` definition, with the name, entity ID and collection of the saved question that holds it."+ [:map {:closed true}+ [:name ms/NonBlankString]+ [:query ::query-definition/query-definition]+ [:entity_id ms/NanoIdString]+ [:collection_id ms/NanoIdString]])++(mr/def ::action+ "An action without a model a data app runs, with the entity ID and collection of its copy."+ [:map {:closed true}+ [:action_id ms/PositiveInt]+ [:entity_id ms/NanoIdString]+ [:collection_id ms/NanoIdString]])++(mr/def ::file+ "A serialization file in the folder of the app's collection, or the error that stops it."+ [:or+ [:map {:closed true}+ [:file :string]+ [:yaml :string]]+ [:map {:closed true}+ [:error :string]]])++(mr/def ::app-request+ "What a new data app's files are generated from."+ [:map {:closed true}+ [:name ::display-name]+ [:slug {:optional true} [:maybe ::slug]]+ [:description {:optional true} ::description]])++(mr/def ::app-files+ "A new data app's files, each at its path from the repository root."+ [:map {:closed true}+ [:files [:sequential [:map {:closed true}+ [:path :string]+ [:yaml :string]]]]])diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex fe175ca40a0..3d56fb3daa0 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -964,7 +964,7 @@(catch Exception _ nil)))}))(defn- dependency->incremental-export-plan [snapshot opts [row entity]]- (let [path (source/entity->path opts entity)]+ (let [path (serialization/entity-file-path opts entity)](if (path-free? (entity-path-info snapshot path (:entity_id entity))){:writes [(assoc row :file_path path)]}:remote-sync/incremental-not-possible)))@@ -1080,7 +1080,7 @@:remote-sync/incremental-not-possible ; extract-chunk omits gone entities; some row is unsyncable(->> found(map (fn [[row entity]]- (row->incremental-export-plan row (entity-path-info snapshot (source/entity->path opts entity) (:entity_id entity)))))+ (row->incremental-export-plan row (entity-path-info snapshot (serialization/entity-file-path opts entity) (:entity_id entity)))))(reduce merge-incremental-export-plans-reducer {})))))(defn- incremental-export-plan@@ -1123,7 +1123,7 @@(not (settings/library-is-remote-synced?)) (into ["snippets" "glossary"])))(defn- stage-write [commit opts [row entity]]- (let [path (or (:file_path row) (source/entity->path opts entity))+ (let [path (or (:file_path row) (serialization/entity-file-path opts entity))fspec (source/entity->file-spec-at path entity)](run! #(source.p/stage-upsert! commit %) (source/file-specs fspec))(when (:id row)diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.cljindex 0154352e44e..07e9831e193 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj@@ -68,20 +68,6 @@(atom nil) (atom []))(seq root-dependencies) (ingestable/wrap-root-dep-ingestable root-dependencies)))-(defn entity->path- "The repo-relative path an extracted `entity` serializes to, using storage context `opts`."- [opts entity]- (let [resolved (serialization/resolve-storage-path opts entity)- dirnames (drop-last resolved)- basename (str (last resolved) ".yaml")]- (str/join "/" (concat dirnames [basename]))))--(defn entity->content- "The serialized YAML string for an extracted `entity`."- [entity]- (yaml/generate-string (serialization/serialization-deep-sort (serialization/without-resources entity))- {:dumper-options {:flow-style :block :split-lines false}}))-(defn- resource-specs"The `{:path :content}` file specs of `entity`'s resource files, next to its YAML file at `path`."[path entity]@@ -94,13 +80,13 @@`:resources` holds the `{:path :content}` specs of its resource files."[path entity]{:path path- :content (entity->content entity)+ :content (serialization/entity-yaml entity):resources (vec (resource-specs path entity))})(defn entity->file-spec"[[entity->file-spec-at]] the path storage context `opts` (from [[serdes/storage-base-context]]) gives `entity`."[opts entity]- (entity->file-spec-at (entity->path opts entity) entity))+ (entity->file-spec-at (serialization/entity-file-path opts entity) entity))(defn file-specs"The `{:path :content}` specs of every file a `file-spec` from [[entity->file-spec]] writes."diff --git a/enterprise/backend/src/metabase_enterprise/serialization/core.clj b/enterprise/backend/src/metabase_enterprise/serialization/core.cljindex ec11ccfe9f9..e51d4c4d67b 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/core.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/core.clj@@ -45,7 +45,10 @@[metabase-enterprise.serialization.v2.storage.filesfile-writer][metabase-enterprise.serialization.v2.storage.util+ entity-file-path+ entity-yamlresolve-storage-pathresource-filesslugify-name- without-resources])+ without-resources+ yaml-file-path])diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.cljindex 06fb6485ef8..304aa9c83f1 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj@@ -13,9 +13,6 @@(set! *warn-on-reflection* true)-(defn- yaml-file ^File [root-dir resolved]- (apply io/file root-dir (concat (drop-last resolved) [(str (last resolved) ".yaml")])))-(defn file-writer"Create a filesystem storage backend rooted at `root-dir`."[root-dir]@@ -23,7 +20,7 @@(reify protocols/ExportWriter(store-entity! [_ entity](let [resolved (storage.util/resolve-storage-path ctx entity)- f (yaml-file root-dir resolved)]+ ^File f (io/file root-dir (storage.util/yaml-file-path resolved))](log/info "Storing" {:path (serdes/log-path-str (:serdes/meta entity)):file (str (.relativize (Path/of (str root-dir) (make-array String 0))(.toPath f)))})diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.cljindex 277218f5331..ab812c1c1b0 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj@@ -23,11 +23,6 @@(.write tar-out content)(.closeArchiveEntry tar-out)))-(defn- entry-path- "Build the tar entry path: dirname/resolved.../name.yaml"- ^String [dirname resolved]- (str dirname "/" (str/join "/" (drop-last resolved)) "/" (last resolved) ".yaml"))-(defn tar-writer"Create a streaming tar.gz storage backend writing to `output`."[^OutputStream output ^String dirname]@@ -40,8 +35,8 @@(reify protocols/ExportWriter(store-entity! [_ entity](let [resolved (storage.util/resolve-storage-path ctx entity)- path (entry-path dirname resolved)- content (.getBytes ^String (yaml-content (storage.util/without-resources entity)) "UTF-8")]+ path (str dirname "/" (storage.util/yaml-file-path resolved))+ content (.getBytes ^String (storage.util/entity-yaml entity) "UTF-8")](log/trace "Storing" {:path (serdes/log-path-str (:serdes/meta entity))})(put-entry! tar path content)(doseq [[segments ^String resource] (storage.util/resource-files resolved entity)]diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.cljindex cb5272f097e..98fa01c52ab 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.serialization.v2.storage.util(:require[clojure.string :as str]+ [metabase-enterprise.serialization.dump :as dump][metabase.lib.core :as lib][metabase.models.serialization :as serdes][metabase.util :as u]@@ -51,11 +52,26 @@[ctx entity](resolve-path (:unique-name-fns ctx) (serdes/storage-path entity ctx)))+(defn yaml-file-path+ "The `/`-separated path of the YAML file at the `resolved` storage path."+ [resolved]+ (str/join "/" (concat (drop-last resolved) [(str (last resolved) ".yaml")])))++(defn entity-file-path+ "The path, relative to the export root, of the YAML file the extracted `entity` serializes to in storage context `ctx`."+ [ctx entity]+ (yaml-file-path (resolve-storage-path ctx entity)))+(defn without-resources"The `entity` as written to its YAML file: [[serdes/storable]], without its `:serdes/resources`."[entity](serdes/storable (dissoc entity :serdes/resources)))+(defn entity-yaml+ "The text of the YAML file the extracted `entity` serializes to."+ [entity]+ (dump/yaml-content (without-resources entity)))+(defn resource-files"Each of `entity`'s `:serdes/resources` as `[path-segments content]`, next to its YAML file at `resolved`."[resolved entity]diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/api_test.cljindex 71cfe1f3c13..e5d51510261 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/api_test.clj@@ -118,8 +118,7 @@(deftest data-app-write-endpoints-require-feature-token-test(mt/with-premium-features #{}- (mt/user-http-request :crowberto :post 402 "apps/serialize-resources"- {:collection "goodAppCollection0000" :queries [] :actions []})))+ (mt/user-http-request :crowberto :post 402 "apps/generate/resources" {:queries [] :actions []})))(deftest assigned-group-reaches-only-copied-actions-test(testing "an action is reachable exactly when it lives in the data app collection"diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/app_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/app_test.cljnew file mode 100644index 00000000000..237a7547991--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/app_test.clj@@ -0,0 +1,83 @@+(ns metabase-enterprise.data-apps.generate.app-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.data-apps.config :as data-apps.config]+ [metabase-enterprise.data-apps.generate.app :as generate.app]+ [metabase-enterprise.data-apps.test-util :as data-apps.tu]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [metabase.util.yaml :as yaml]))++(use-fixtures :once (fixtures/initialize :db :web-server :test-users))++(defn- file-yaml+ "The parsed YAML of the file at `path` among the `files` of a generated app."+ [{:keys [files]} path]+ (some-> (first (filter #(= path (:path %)) files)) :yaml yaml/parse-string))++(deftest generates-the-manifest-and-the-collection-file-test+ (let [result (generate.app/generate {:name "Sales Ops"})+ app (file-yaml result "data_apps/sales-ops/data_app.yaml")+ collection-id (:entity_id (file-yaml result (data-apps.tu/collection-path "Data App: sales-ops")))]+ (testing "a manifest and a collection file, each at the path a remote-sync export writes it"+ (is (= ["data_apps/sales-ops/data_app.yaml" "collections/data_apps/data_app__sales_ops.yaml"]+ (map :path (:files result)))))+ (testing "the manifest names the app, its bundle, its version and the collection it owns"+ (is (=? {:name "Sales Ops"+ :slug "sales-ops"+ :path "./dist/index.js"+ :version data-apps.config/supported-app-version+ :collection collection-id+ :entity_id string?+ :serdes/meta [{:model "DataApp"}]}+ app))+ (is (not (contains? app :description))))+ (testing "the collection is in the data-apps namespace, named after the app's slug as the instance names it"+ (is (=? {:name "Data App: sales-ops"+ :namespace "data-apps"+ :entity_id string?+ :serdes/meta [{:model "Collection"}]}+ (file-yaml result "collections/data_apps/data_app__sales_ops.yaml"))))+ (testing "the entity IDs are new on each call"+ (is (not= (:entity_id app)+ (:entity_id (file-yaml (generate.app/generate {:name "Sales Ops"})+ "data_apps/sales-ops/data_app.yaml")))))))++(deftest keeps-the-description-and-an-explicit-slug-test+ (let [result (generate.app/generate {:name "Sales Ops" :slug "ops" :description "Pipeline health"})]+ (is (=? {:slug "ops" :description "Pipeline health"}+ (file-yaml result "data_apps/ops/data_app.yaml")))+ (testing "the collection is named after the slug, so apps sharing a display name don't share a collection file"+ (is (=? {:name "Data App: ops"} (file-yaml result "collections/data_apps/data_app__ops.yaml"))))))++(deftest derives-the-slug-from-the-name-test+ (are [app-name slug] (some? (file-yaml (generate.app/generate {:name app-name})+ (str "data_apps/" slug "/data_app.yaml")))+ "Sales Ops" "sales-ops"+ " Q3 -- Revenue!" "q3-revenue"+ "inventory 2" "inventory-2"))++(deftest refuses-a-name-without-a-valid-slug-test+ (doseq [app-name ["???" "Generate"]]+ (is (= 400 (:status-code (ex-data (is (thrown? clojure.lang.ExceptionInfo+ (generate.app/generate {:name app-name}))))))+ app-name))+ (testing "an explicit slug is checked too"+ (is (thrown? clojure.lang.ExceptionInfo (generate.app/generate {:name "Sales" :slug "Not A Slug"})))))++(deftest api-generates-the-files-test+ (mt/with-premium-features #{:data-apps}+ (testing "a superuser gets the files"+ (is (=? {:files [{:path "data_apps/sales-ops/data_app.yaml" :yaml string?}+ {:path "collections/data_apps/data_app__sales_ops.yaml" :yaml string?}]}+ (mt/user-http-request :crowberto :post 200 "apps/generate/app" {:name "Sales Ops"}))))+ (testing "an explicit slug is kept"+ (is (=? {:files [{:path "data_apps/ops/data_app.yaml"} {}]}+ (mt/user-http-request :crowberto :post 200 "apps/generate/app" {:name "Sales Ops" :slug "ops"}))))+ (testing "only a superuser writes an app's repository"+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :post 403 "apps/generate/app" {:name "Sales Ops"}))))+ (testing "a name that gives no valid slug is refused"+ (mt/user-http-request :crowberto :post 400 "apps/generate/app" {:name "???"}))+ (testing "a reserved slug is refused"+ (mt/user-http-request :crowberto :post 400 "apps/generate/app" {:name "Sales Ops" :slug "generate"}))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.cljnew file mode 100644index 00000000000..198bee029f8--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj@@ -0,0 +1,529 @@+(ns metabase-enterprise.data-apps.generate.resources-test+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [clojure.walk :as walk]+ [medley.core :as m]+ [metabase-enterprise.data-apps.generate.resources :as generate.resources]+ [metabase-enterprise.data-apps.test-util :as data-apps.tu]+ [metabase.actions.core :as actions]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.models.serialization :as serdes]+ [metabase.test :as mt]+ [metabase.util :as u]+ [metabase.util.malli.fn :as mu.fn]+ [metabase.util.yaml :as yaml]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^:private app-collection "appCollectionEntity01")++(def ^:private other-collection "otherCollectionEnt001")++(defn- do-with-collections!+ "Call `f` with the collections `app-collection` and `other-collection` in place."+ [f]+ (mt/with-temp [:model/Collection _ {:entity_id app-collection :name "App collection" :namespace "data-apps"}+ :model/Collection _ {:entity_id other-collection :name "Other collection" :namespace "data-apps"}]+ (f)))++(defn- query-item+ "A query to serialize named `query-name`, in `app-collection` with a fresh entity ID unless `extra` says otherwise."+ [query-name definition & {:as extra}]+ (merge {:name query-name :query definition :entity_id (u/generate-nano-id) :collection_id app-collection} extra))++(defn- action-item+ "An action to serialize, in `app-collection` with a fresh entity ID unless `action` is a map saying otherwise."+ [action]+ (merge {:entity_id (u/generate-nano-id) :collection_id app-collection}+ (if (map? action) action {:action_id action})))++(defn- request-body [{:keys [queries actions]}]+ {:queries (vec queries) :actions (mapv action-item actions)})++(defn- generate!+ "Generate the files of `body` as `user` over HTTP, with the collections in place."+ ([user status body]+ (generate! user status body #{:data-apps}))+ ([user status body features]+ (do-with-collections!+ (fn []+ (mt/with-premium-features features+ (mt/user-http-request user :post status "apps/generate/resources" (request-body body)))))))++(defn- generate-directly!+ "Generate the files of `body` as a superuser by calling the function, for a test that redefines what runs on the request thread,+ with the types of its definitions as keywords, as the endpoint hands them over."+ [body]+ (do-with-collections!+ (fn []+ (mt/with-premium-features #{:data-apps}+ (mt/with-current-user (mt/user->id :crowberto)+ (generate.resources/generate+ (walk/postwalk #(cond-> % (and (map? %) (string? (:type %))) (update :type keyword))+ (request-body body))))))))++(defn- file-entity+ "The entity a serialization `file` holds."+ [{:keys [yaml]}]+ (yaml/parse-string yaml))++(defn- db-name []+ (t2/select-one-fn :name :model/Database (mt/id)))++(defn- table-path [table]+ [(db-name) "PUBLIC" table])++(defn- field-path [table field]+ [(db-name) "PUBLIC" table field])++(defn- venues-column [field-name & {:as extra}]+ (merge {:type "column" :name field-name :tableId (mt/id :venues) :sourceName "VENUES"} extra))++(defn- venues-definition [& {:as stage}]+ {:stages [(merge {:source {:type "table" :id (mt/id :venues)}} stage)]})++(defn- metric-count-definition [metric-id]+ (venues-definition :aggregations [{:type "metric" :id metric-id}]))++(deftest serialization-needs-each-items-entity-id-name-and-collection-test+ (testing "the saved question is written with the given entity ID, name and collection, so every query names them"+ (mt/with-premium-features #{:data-apps}+ (doseq [missing [:name :entity_id :collection_id]]+ (testing (pr-str missing)+ (is (=? {:errors {:queries some?}}+ (mt/user-http-request :crowberto :post 400 "apps/generate/resources"+ {:queries [(dissoc (query-item "Venues" (venues-definition)) missing)]+ :actions []}))))))))++(deftest serialization-needs-each-actions-entity-id-and-collection-test+ (mt/with-premium-features #{:data-apps}+ (doseq [missing [:entity_id :collection_id]]+ (testing (pr-str missing)+ (is (=? {:errors {:actions some?}}+ (mt/user-http-request :crowberto :post 400 "apps/generate/resources"+ {:queries []+ :actions [(dissoc (action-item 1) missing)]})))))))++(deftest serializes-into-a-collection-absent-from-the-instance-test+ (data-apps.tu/do-with-sources!+ (fn [{:keys [action-id]}]+ (let [response (generate! :crowberto 200+ {:queries [(query-item "Gone" (venues-definition) :collection_id "noSuchCollectionEnt01")]+ :actions [{:action_id action-id :collection_id "noSuchCollectionEnt01"}]})]+ (is (=? {:queries [{:file "gone.yaml"}] :actions [{:file string?}]} response))+ (doseq [file (concat (:queries response) (:actions response))]+ (is (= "noSuchCollectionEnt01" (:collection_id (file-entity file)))))))))++(deftest names-files-after-their-entities-and-deduplicates-them-test+ (data-apps.tu/do-with-sources!+ (fn [{:keys [metric-id]}]+ (is (=? {:queries [{:file "orders.yaml"} {:file "orders_2.yaml"} {:file "venue_count.yaml"}]+ :metrics [{:file string?}]}+ (generate! :crowberto 200+ {:queries [(query-item "Orders" (venues-definition))+ (query-item "Orders" (venues-definition))+ (query-item "Venue count" (metric-count-definition metric-id))]}))))))++(deftest serializes-a-query-with-every-reference-portable-test+ (let [response (generate! :crowberto 200+ {:queries [(query-item "PriceByCategory"+ (venues-definition+ :filters [{:type "operator" :operator ">"+ :args [(venues-column "PRICE")+ {:type "literal" :value 1}]}]+ :aggregations [{:type "operator" :operator "sum" :name "total"+ :args [(venues-column "PRICE")]}]+ :breakouts [{:type "column"+ :name "NAME"+ :sourceName "CATEGORIES"+ :sourceFieldId (mt/id :venues :category_id)}]+ :orderBys [{:type "column"+ :name "total"+ :direction "desc"}]+ :limit 5))]})+ entity (file-entity (first (:queries response)))]+ (testing "the query Metabase builds from the definition, referencing tables and fields by name"+ (let [price (field-path "VENUES" "PRICE")+ category-id (field-path "VENUES" "CATEGORY_ID")+ category (field-path "CATEGORIES" "NAME")]+ (is (=? {:dataset_query {:lib/type "mbql/query"+ :database (db-name)+ :stages [{:source-table (table-path "VENUES")+ :filters [[">" {} ["field" {} price] 1]]+ :aggregation [["sum" {:name "total"} ["field" {} price]]]+ :breakout [["field" {:source-field category-id} category]]+ :order-by [["desc" {} ["aggregation"+ {:lib/source-name "total"}+ string?]]]+ :limit 5}]}}+ entity))+ (is (empty? (:metrics response)))))+ (testing "only the uuid the order by points at is kept, and it is the aggregation's"+ (let [stage (-> entity :dataset_query :stages first)]+ (is (= (get-in stage [:aggregation 0 1 :lib/uuid])+ (get-in stage [:order-by 0 2 2])))+ (is (nil? (get-in stage [:filters 0 1 :lib/uuid])))))))++(deftest only-a-superuser-generates-test+ (testing "an app's resources are written into its repository, which only an admin works with"+ (let [body {:queries [(query-item "Venues" (venues-definition))]}]+ (is (= "You don't have permissions to do that." (generate! :rasta 403 body)))+ (is (=? {:queries [{:file string? :yaml string?}]} (generate! :crowberto 200 body))))))++(deftest lists-the-metrics-a-query-aggregates-test+ (data-apps.tu/do-with-sources!+ (fn [{:keys [metric-id]}]+ (let [metric-eid (t2/select-one-fn :entity_id :model/Card :id metric-id)+ response (generate! :crowberto 200+ {:queries [(query-item "VenueCount" (metric-count-definition metric-id))]})+ metric (file-entity (first (:metrics response)))+ question (file-entity (first (:queries response)))]+ (is (= 1 (count (:metrics response))))+ (is (=? {:type "metric"+ :collection_id app-collection+ :dataset_query {:stages [{:source-table (table-path "VENUES")}]}}+ metric))+ (testing "the copy has its own entity ID, and the question reads the copy rather than the source metric"+ (is (not= metric-eid (:entity_id metric)))+ (is (= (:entity_id metric) (get-in question [:dataset_query :stages 0 :aggregation 0 2])))+ (is (not= metric-eid (get-in question [:dataset_query :stages 0 :aggregation 0 2]))))+ (testing "the copy's entity ID is the same every time"+ (is (= (:entity_id metric)+ (-> (generate! :crowberto 200 {:queries [(query-item "VenueCount" (metric-count-definition metric-id))]})+ :metrics first file-entity :entity_id))))))))++(deftest copies-a-metric-once-per-collection-test+ (data-apps.tu/do-with-sources!+ (fn [{:keys [metric-id]}]+ (let [definition (metric-count-definition metric-id)+ response (generate! :crowberto 200+ {:queries [(query-item "A" definition)+ (query-item "B" definition)+ (query-item "C" definition :collection_id other-collection)]})+ metrics (mapv file-entity (:metrics response))+ references (mapv #(get-in (file-entity %) [:dataset_query :stages 0 :aggregation 0 2]) (:queries response))]+ (is (= 2 (count metrics)))+ (is (= #{app-collection other-collection} (set (map :collection_id metrics))))+ (is (apply distinct? (map :entity_id metrics)))+ (is (= (first references) (second references)))+ (is (= (set references) (set (map :entity_id metrics))))))))++(deftest answers-each-query-on-its-own-test+ (let [response (generate! :crowberto 200+ {:queries [(query-item "Venues" (venues-definition))+ (query-item "Broken" (venues-definition :fields [(venues-column "NOT_A_COLUMN")]))]})]+ (is (=? [{:file string? :yaml string?} {:error "No column found"}]+ (:queries response)))+ (is (=? {:dataset_query {:stages [{:source-table (table-path "VENUES")}]}}+ (file-entity (first (:queries response)))))))++(deftest serializes-a-query-action-test+ (data-apps.tu/do-with-sources!+ (fn [{:keys [action-id]}]+ (let [response (generate! :crowberto 200 {:actions [{:action_id action-id :entity_id "actionCopyEntity00001"}]})+ entity (file-entity (first (:actions response)))]+ (is (=? {:entity_id "actionCopyEntity00001"+ :collection_id app-collection+ :type "query"+ :query [{:database_id (db-name)+ :dataset_query {:database (db-name)}}]}+ entity))+ (is (not (contains? entity :model_id))+ "it names no model")))))++(deftest refuses-an-action-that-belongs-to-a-model-test+ (testing "a data app runs only actions that belong to no model, as the typed schema lists only those"+ (data-apps.tu/do-with-sources!+ (fn [{:keys [model-action-id]}]+ (is (=? {:actions [{:error #"Action \d+ belongs to a model\..*"}]}+ (generate! :crowberto 200 {:actions [model-action-id]})))))))++(deftest refuses-what-a-copy-cannot-hold-test+ (data-apps.tu/do-with-sources!+ (fn [{:keys [metric-id model-id]}]+ (let [mp (mt/metadata-provider)+ model-count (lib/aggregate (lib/query mp (lib.metadata/card mp model-id)) (lib/count))]+ (mt/with-temp [:model/Card {reading-metric-id :id} {:name "Metric reading a model"+ :type :metric+ :database_id (mt/id)+ :dataset_query model-count}]+ (let [card-sql-id (actions/insert! {:name "Read a card"+ :type :query+ :database_id (mt/id)+ :dataset_query (lib/native-query+ mp (str "SELECT * FROM {{#" metric-id "}}"))})+ response (generate! :crowberto 200+ {:queries [(query-item "ReadsMetric" (metric-count-definition reading-metric-id))]+ :actions [card-sql-id Integer/MAX_VALUE]})]+ (testing "a query action whose SQL reads a card"+ (is (=? {:error (re-pattern (str ".*reads card " metric-id ".*"))}+ (nth (:actions response) 0))))+ (testing "an action that does not exist"+ (is (=? {:error #".*does not exist.*"} (nth (:actions response) 1))))+ (testing "a metric that reads a card; the query that aggregates it is still built"+ (is (=? {:queries [{:file string?}]+ :metrics [{:error (re-pattern (str ".*reads card " model-id ".*"))}]}+ response)))))))))++(deftest rejects-unsupported-definitions-test+ (testing "the request accepts only what data app definitions support"+ (doseq [path [[:queries 0 :query :stages 0 :joins]+ [:queries 0 :query :stages 0 :expressions]]]+ (testing (pr-str path)+ (do-with-collections!+ (fn []+ (mt/with-premium-features #{:data-apps}+ (mt/user-http-request :crowberto :post 400 "apps/generate/resources"+ (assoc-in {:queries [(query-item "Venues" (venues-definition))] :actions []}+ path [])))))))))++(deftest refuses-a-table-that-does-not-exist-test+ (is (=? {:queries [{:error (str "Table " Integer/MAX_VALUE " does not exist.")}]}+ (generate! :crowberto 200+ {:queries [(query-item "Nothing" {:stages [{:source {:type "table" :id Integer/MAX_VALUE}}]})]})))+ (testing "a deactivated table is gone too, as it is from the typed schema"+ (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :active false}]+ (is (=? {:queries [{:error (str "Table " table-id " does not exist.")}]}+ (generate! :crowberto 200+ {:queries [(query-item "Gone" {:stages [{:source {:type "table" :id table-id}}]})]}))))))++(deftest refuses-a-column-that-reaches-a-deactivated-table-test+ (testing "types generated before categories was deactivated still select its name through venues.category_id, and+ the query would fail on the implicit join when it runs"+ (mt/with-temp-vals-in-db :model/Table (mt/id :categories) {:active false}+ (is (=? {:queries [{:error (str "Table " (mt/id :categories) " does not exist.")}]}+ (generate! :crowberto 200+ {:queries [(query-item "VenueCategories"+ (venues-definition+ :fields [{:type "column" :name "NAME"+ :sourceFieldId (mt/id :venues :category_id)}]))]}))))))++(deftest refuses-a-metric-that-reaches-a-deactivated-table-test+ (testing "a metric the query aggregates reads categories through venues.category_id in its own query, which the+ built query holds only the ID of, and the query would fail on the table when it runs"+ (let [mp (mt/metadata-provider)+ venues (lib/query mp (lib.metadata/table mp (mt/id :venues)))+ categories (mt/id :categories)+ category (m/find-first (comp #{(mt/id :categories :name)} :id) (lib/filterable-columns venues))]+ (mt/with-temp [:model/Card {bars-id :id} {:name "Bars"+ :type :metric+ :database_id (mt/id)+ :dataset_query (-> venues+ (lib/filter (lib/= category "Bar"))+ (lib/aggregate (lib/count)))}+ :model/Card {count-id :id} {:name "Venue count"+ :type :metric+ :database_id (mt/id)+ :dataset_query (lib/aggregate venues (lib/count))}]+ (mt/with-temp-vals-in-db :model/Table categories {:active false}+ (is (=? {:queries [{:error (str "Table " categories " does not exist.")}+ {:file string?}]+ :metrics [{:file string?}]}+ (generate! :crowberto 200+ {:queries [(query-item "BarCount" (metric-count-definition bars-id))+ (query-item "VenueCount" (metric-count-definition count-id))]}))))))))++(deftest refuses-a-column-remapped-to-a-deactivated-table-test+ (testing "venues.category_id displays categories.name, a join the query processor adds only when the query runs, so+ the built query never names categories"+ (let [categories (mt/id :categories)]+ (mt/with-column-remappings [venues.category_id categories.name]+ (mt/with-temp-vals-in-db :model/Table categories {:active false}+ (is (=? {:queries [{:error (str "Table " categories " does not exist.")}+ {:file string?}]}+ (generate! :crowberto 200+ {:queries [(query-item "VenueCategoryIds"+ (venues-definition :fields [{:type "column" :name "CATEGORY_ID"}]))+ (query-item "VenuePrices"+ (venues-definition :fields [{:type "column" :name "PRICE"}]))]}))))))))++(deftest refuses-a-definition-that-builds-an-invalid-query-test+ (testing "the request schema accepts what a type lets through, and lib's own checks are off in production, so the+ built query is checked: an invalid one must not serialize and then fail when it runs"+ (binding [mu.fn/*enforce* false]+ (is (=? {:queries [{:error "The definition does not build a valid query."}+ {:file string?}]}+ (generate-directly!+ {:queries [(query-item "Half" {:stages [{:source {:type :table :id (mt/id :venues)} :limit 1.5}]})+ (query-item "Whole" {:stages [{:source {:type :table :id (mt/id :venues)} :limit 2}]})]}))))))++(deftest a-public-source-serializes-as-the-private-copy-the-author-writes-test+ (testing "what makes a source public or embedded is left out, since the pull refuses a copy that says it is"+ (data-apps.tu/do-with-sources!+ (fn [{:keys [metric-id action-id]}]+ (t2/update! :model/Card :id metric-id {:public_uuid (str (random-uuid))+ :made_public_by_id (mt/user->id :crowberto)+ :enable_embedding true+ :embedding_params {}})+ (t2/update! :model/Action :id action-id {:public_uuid (str (random-uuid))+ :made_public_by_id (mt/user->id :crowberto)})+ (let [{:keys [actions metrics] :as response}+ (generate! :crowberto 200+ {:queries [(query-item "VenueCount" (metric-count-definition metric-id))]+ :actions [action-id]})]+ (is (=? {:actions [{:file string?}] :metrics [{:file string?}]} response))+ (doseq [file (concat actions metrics)]+ (is (not-any? (partial contains? (file-entity file))+ [:public_uuid :made_public_by_id :enable_embedding :embedding_params :embedding_type]))))))))++(deftest an-item-serialization-cannot-extract-answers-with-the-cause-test+ (testing "one entity that fails inside its extraction comes back with the reason, which serialization wraps at each+ level, and the rest still serialize"+ (data-apps.tu/do-with-sources!+ (fn [{:keys [metric-id action-id]}]+ (t2/update! :model/Card :id metric-id {:visualization_settings {:broken true}})+ (let [export-settings (mt/original-fn #'serdes/export-visualization-settings)]+ (mt/with-dynamic-fn-redefs [serdes/export-visualization-settings (fn [settings]+ (if (:broken settings)+ (throw (ex-info "the metric is broken" {}))+ (export-settings settings)))]+ (is (=? {:queries [{:file string?}]+ :actions [{:file string?}]+ :metrics [{:error (str "Could not serialize Metric " metric-id ": the metric is broken")}]}+ (generate-directly!+ {:queries [(query-item "VenueCount" (metric-count-definition metric-id))]+ :actions [action-id]})))))))))++(deftest an-item-serialization-leaves-out-answers-without-a-reason-test+ (testing "a card materialized by an exploration Summary is one serialization leaves out without an error"+ (data-apps.tu/do-with-sources!+ (fn [{:keys [metric-id]}]+ (mt/with-temp [:model/Exploration {exploration-id :id} {:name "Explo" :creator_id (mt/user->id :crowberto)}+ :model/Document {summary-id :id} {:name "Summary"+ :creator_id (mt/user->id :crowberto)+ :exploration_id exploration-id}]+ (t2/update! :model/Card :id metric-id {:document_id summary-id})+ (is (=? {:metrics [{:error (str "Could not serialize Metric " metric-id ".")}]}+ (generate! :crowberto 200+ {:queries [(query-item "VenueCount" (metric-count-definition metric-id))]}))))))))++(deftest a-failure-of-the-serialization-is-logged-and-a-refusal-is-not-test+ (testing "a refusal is the author's to act on; anything else is a failure the server keeps a trace of"+ (mt/with-log-messages-for-level [messages [metabase-enterprise.data-apps.generate.resources :warn]]+ (is (=? {:queries [{:error "No column found"}+ {:error (str "Table " Integer/MAX_VALUE " does not exist.")}]}+ (generate-directly!+ {:queries [(query-item "Broken"+ {:stages [{:source {:type :table :id (mt/id :venues)}+ :fields [{:type :column :name "NOT_A_COLUMN"}]}]})+ (query-item "Nothing" {:stages [{:source {:type :table :id Integer/MAX_VALUE}}]})]})))+ (let [logged (filter #(str/includes? (:message %) "Could not serialize a data app resource") (messages))]+ (is (= 1 (count logged)))+ (testing "naming the query by its entity ID, without its definition"+ (is (str/includes? (:message (first logged)) ":query"))+ (is (not (str/includes? (:message (first logged)) "NOT_A_COLUMN")))+ (is (not (str/includes? (:message (first logged)) "Broken"))))))))++(deftest refuses-archived-sources-test+ (testing "the pull refuses an archived resource, so the serialization refuses an archived source"+ (data-apps.tu/do-with-sources!+ (fn [{:keys [action-id]}]+ (t2/update! :model/Action :id action-id {:archived true})+ (is (=? {:actions [{:error #".*is archived.*"}]}+ (generate! :crowberto 200 {:actions [action-id]})))))))++(deftest refuses-an-action-whose-parameter-values-come-from-a-card-test+ (data-apps.tu/do-with-sources!+ (fn [{:keys [metric-id]}]+ (let [mp (mt/metadata-provider)+ action-id (actions/insert! {:name "Pick"+ :type :query+ :database_id (mt/id)+ :dataset_query (lib/native-query mp "UPDATE venues SET name = {{name}}")+ :parameters [{:id "name" :slug "name" :type :string/=+ :values_source_type :card+ :values_source_config {:card_id metric-id}}]})]+ (is (=? {:actions [{:error (re-pattern (str ".*reads card " metric-id ".*"))}]}+ (generate! :crowberto 200 {:actions [action-id]})))))))++(deftest serializes-a-segment-reference-by-entity-id-test+ (let [mp (mt/metadata-provider)]+ (mt/with-temp [:model/Segment {segment-id :id, segment-eid :entity_id}+ {:name "Cheap"+ :table_id (mt/id :venues)+ :definition (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))+ (lib/filter (lib/< (lib.metadata/field mp (mt/id :venues :price)) 3)))}]+ (is (=? {:dataset_query {:stages [{:filters [["segment" {} segment-eid]]}]}}+ (-> (generate! :crowberto 200+ {:queries [(query-item "Cheap"+ (venues-definition :filters [{:type "segment" :id segment-id}]))]})+ :queries first file-entity))))))++(deftest refuses-sources-on-a-routing-destination-test+ (testing "a routing destination is reachable only through its router, so the typed schema leaves out what it backs,+ and so does the serialization"+ (mt/with-temp [:model/Database {router-id :id} {}+ :model/DatabaseRouter _ {:database_id router-id :user_attribute "region"}+ :model/Database {destination-id :id} {:router_database_id router-id}]+ (let [action-id (actions/insert! {:name "Rename venue"+ :type :query+ :database_id destination-id+ :dataset_query {:lib/type :mbql/query+ :database destination-id+ :stages [{:lib/type :mbql.stage/native+ :native "UPDATE venues SET name = 'x'"}]}})]+ (is (=? {:actions [{:error #".*is backed by a routing destination.*"}]}+ (generate! :crowberto 200 {:actions [action-id]})))))))++(deftest refuses-a-source-whose-settings-read-a-card-test+ (testing "a copy keeps every card its serialization references, so a click behaviour linking a saved question is refused+ like a query reading one"+ (data-apps.tu/do-with-sources!+ (fn [{:keys [metric-id model-id]}]+ (t2/update! :model/Card :id metric-id+ {:visualization_settings {:click_behavior {:type "link" :linkType "question" :targetId model-id}}})+ (is (=? {:metrics [{:error (re-pattern (str ".*reads card " model-id ".*"))}]}+ (generate! :crowberto 200+ {:queries [(query-item "VenueCount" (metric-count-definition metric-id))]})))))))++(deftest extracts-the-cards-and-the-actions-once-test+ (testing "the cards and the actions are each extracted in one serialization query, however many the app uses"+ (data-apps.tu/do-with-sources!+ (fn [{:keys [metric-id action-id model-action-id]}]+ (let [calls (atom [])+ extract (mt/original-fn #'generate.resources/extract-by-entity-id)]+ (mt/with-dynamic-fn-redefs [generate.resources/extract-by-entity-id (fn [model-name ids]+ (swap! calls conj model-name)+ (extract model-name ids))]+ (is (=? {:actions [{:file string?} {:error string?}] :metrics [{:file string?}]}+ (generate-directly!+ {:queries [(query-item "VenueCount" (metric-count-definition metric-id))]+ :actions [action-id model-action-id]}))))+ (is (= {"Card" 1 "Action" 1} (frequencies @calls))))))))++(deftest serializes-the-saved-question-an-author-writes-test+ (testing "a query comes back as the saved question that holds it: named as given, in the given collection, with the+ given entity ID, created by the caller, and nothing unset"+ (let [{:keys [file yaml] :as item} (-> (generate! :crowberto 200+ {:queries [(query-item "Venues list" (venues-definition :limit 5)+ :entity_id "savedQuestionEntity01")]})+ :queries first)+ entity (file-entity item)]+ (is (= "venues_list.yaml" file))+ (is (string? yaml))+ (is (=? {:serdes/meta [{:model "Card"}]+ :entity_id "savedQuestionEntity01"+ :collection_id app-collection+ :name "Venues list"+ :type "question"+ :display "table"+ :creator_id "[email protected]"+ :visualization_settings {:column_settings nil}+ :parameters []+ :parameter_mappings []+ :dataset_query {:database (db-name)+ :stages [{:source-table (table-path "VENUES") :limit 5}]}}+ entity))+ (is (not-any? (partial contains? entity)+ [:description :collection_position :public_uuid :card_schema :archived :enable_embedding])+ "what serialization leaves unset or at its default is left out, as the format omits it"))))++(deftest keys-come-in-the-order-serialization-writes-them-test+ (testing "the printed entity comes in the order serialization writes a file, so an author keeps it"+ (let [{:keys [yaml] :as file} (-> (generate-directly! {:queries [(query-item "Venues" (venues-definition))]})+ :queries first)]+ (is (= [:database :stages :lib/type] (keys (:dataset_query (file-entity file)))))+ (is (< (str/index-of yaml "name:") (str/index-of yaml "\ndataset_query:"))))))diff --git a/test/metabase/typed_schemas/schema/action_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/action_test.cljsimilarity index 86%rename from test/metabase/typed_schemas/schema/action_test.cljrename to enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/action_test.cljindex a38332277e5..879a50a0780 100644--- a/test/metabase/typed_schemas/schema/action_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/action_test.clj@@ -1,18 +1,18 @@-(ns metabase.typed-schemas.schema.action-test+(ns metabase-enterprise.data-apps.generate.schemas.action-test(:require[clojure.test :refer :all]+ [metabase-enterprise.data-apps.generate.schemas.action :as schemas.action][metabase.actions.core :as actions][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]- [metabase.test :as mt]- [metabase.typed-schemas.schema.action :as schema.action]))+ [metabase.test :as mt]))(defn- standalone-action-schema"The schema entry for a model-less query action created from `action` and `dataset-query`."[action dataset-query](mt/with-temp [:model/Action {action-id :id} (merge {:type :query, :name "Update bird"} action):model/QueryAction _ {:action_id action-id, :dataset_query dataset-query}]- (schema.action/action-schema (actions/select-action :id action-id))))+ (schemas.action/action-schema (actions/select-action :id action-id))))(deftest action-schema-types-a-parameter-from-its-template-tag-test(testing ":category has no JS type of its own, so the persisted template tag's type decides it"@@ -44,13 +44,13 @@(deftest action-schema-tolerates-an-empty-query-test(testing "an action whose stored query degraded to {} still builds, typing parameters from their own types"(is (=? {:parameters [{:slug "name", :displayName "Name", :jsType "string"}]}- (schema.action/action-schema {:id 7, :name "Update bird", :dataset_query {}- :parameters [{:id "name", :name "Name", :type :text}]})))))+ (schemas.action/action-schema {:id 7, :name "Update bird", :dataset_query {}+ :parameters [{:id "name", :name "Name", :type :text}]})))))(deftest action-schema-leaves-out-hidden-parameters-test(testing "a parameter the action's form hides is left out, since execute refuses a value for it"(is (=? {:parameters [{:slug "name"}]}- (schema.action/action-schema+ (schemas.action/action-schema{:id 7:name "Update bird":dataset_query {}@@ -58,7 +58,7 @@{:id "updated_by", :name "Updated by", :type :text, :required true}]:visualization_settings {:fields {"updated_by" {:id "updated_by", :hidden true}}}})))(is (= ["name"]- (map :slug (:parameters (schema.action/action-schema+ (map :slug (:parameters (schemas.action/action-schema{:id 7, :name "Update bird", :dataset_query {}:parameters [{:id "name", :type :text} {:id "updated_by", :type :text}]:visualization_settings {:fields {"updated_by" {:id "updated_by", :hidden true}}}})))))))diff --git a/test/metabase/typed_schemas/common_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/common_test.cljsimilarity index 74%rename from test/metabase/typed_schemas/common_test.cljrename to enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/common_test.cljindex c9aee8fdd94..44b9d7c4179 100644--- a/test/metabase/typed_schemas/common_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/common_test.clj@@ -1,12 +1,13 @@-(ns metabase.typed-schemas.common-test+(ns metabase-enterprise.data-apps.generate.schemas.common-test(:require[clojure.test :refer :all]+ [metabase-enterprise.data-apps.db :as data-apps.db]+ [metabase-enterprise.data-apps.generate.schemas.common :as schemas.common][metabase.audit-app.core :as audit][metabase.audit-app.impl :as audit.impl][metabase.config.core :as config][metabase.test :as mt]- [metabase.typed-schemas.common :as typed-schemas.common]- [metabase.typed-schemas.schema.common :as schema.common]))+ [toucan2.core :as t2]))(deftest column-schema-includes-description-test(is (= {:type "column"@@ -15,14 +16,14 @@:baseType "type/Text":jsType "string":description "Name of the customer"}- (typed-schemas.common/column-schema {:name "name"- :display_name "Name"- :base_type "type/Text"- :description "Name of the customer"}))))+ (schemas.common/column-schema {:name "name"+ :display_name "Name"+ :base_type "type/Text"+ :description "Name of the customer"}))))(deftest column-schema-maps-metabase-types-test(are [column js-type] (= js-type- (:jsType (typed-schemas.common/column-schema column)))+ (:jsType (schemas.common/column-schema column))){:name "bool", :base_type :type/Boolean} "boolean"{:name "int", :base_type :type/Integer} "number"{:name "date", :base_type :type/DateTime} "Date"@@ -39,7 +40,7 @@"channelOrders" {:key "channelOrders":id "ca9bef16-d484-4add-8245-ddbc78287e8f":tableId 167}}- (typed-schemas.common/keyed-map+ (schemas.common/keyed-map[{:key "channel":id "ca9bef16-d484-4add-8245-ddbc78287e8f":tableId 167@@ -56,7 +57,7 @@"channelOrders2" {:key "channelOrders2":id 2:tableId 168}}- (typed-schemas.common/keyed-map+ (schemas.common/keyed-map[{:key "channel":id 1:tableId 167@@ -66,25 +67,26 @@:tableId 168:keyDisambiguator "Orders"}]))))-(deftest destination-db-ids-test+(deftest destination-database-ids-test(testing "returns only the ids that back a destination (routed) database"(mt/with-temp [:model/Database {router-id :id} {}:model/Database {destination-id :id} {:router_database_id router-id}:model/Database {open-id :id} {}](is (= #{destination-id}- (schema.common/destination-db-ids #{router-id destination-id open-id})))))- (testing "returns nil for an empty input, without querying"- (is (nil? (schema.common/destination-db-ids #{})))- (is (nil? (schema.common/destination-db-ids nil)))))+ (data-apps.db/destination-database-ids #{router-id destination-id open-id})))))+ (testing "returns an empty set for an empty input"+ (is (= #{} (data-apps.db/destination-database-ids #{})))+ (is (= #{} (data-apps.db/destination-database-ids nil)))))-(deftest select-schema-cards-excludes-destination-database-cards-test- (testing "select-schema-cards excludes cards backed by a destination (routed) database, even for a superuser"+(deftest without-unavailable-cards-excludes-destination-database-cards-test+ (testing "cards backed by a destination (routed) database are left out, even for a superuser"(mt/with-temp [:model/Database {router-id :id} {}:model/Database {destination-id :id} {:router_database_id router-id}:model/Card {open-card-id :id} {:type :metric :database_id (mt/id)}:model/Card {destination-card-id :id} {:type :metric :database_id destination-id}](mt/with-test-user :crowberto- (let [card-ids (into #{} (map :id) (schema.common/select-schema-cards :metric nil nil))]+ (let [cards (t2/select :model/Card :id [:in [open-card-id destination-card-id]])+ card-ids (into #{} (map :id) (schemas.common/without-unavailable-cards cards))](is (contains? card-ids open-card-id))(is (not (contains? card-ids destination-card-id))))))))@@ -97,16 +99,16 @@(testing "a card in the audit collection 403s on its details lookup and one on the audit database has no tablewhile the feature is off, so the schema leaves both out rather than fail an unscoped request"(mt/with-premium-features #{}- (is (= [1] (map :id (#'schema.common/without-unavailable-cards cards))))))+ (is (= [1] (map :id (schemas.common/without-unavailable-cards cards))))));; a premium feature is never on without the EE code, whatever the token says(when config/ee-available?(testing "with the feature on they are listed"(mt/with-premium-features #{:audit-app}- (is (= [1 2 3] (map :id (#'schema.common/without-unavailable-cards cards))))))))))+ (is (= [1 2 3] (map :id (schemas.common/without-unavailable-cards cards))))))))))(deftest ^:parallel keyed-map-never-drops-an-entity-test(testing "a disambiguated key that equals another entity's own key is disambiguated again rather than overwriting it"(let [entities [{:key "step", :id 1} {:key "step", :id 23} {:key "step1", :id 7} {:key "step11", :id 40}]- result (typed-schemas.common/keyed-map entities)]+ result (schemas.common/keyed-map entities)](is (= 4 (count result)))(is (= #{1 23 7 40} (into #{} (map :id) (vals result)))))))diff --git a/test/metabase/typed_schemas/javascript_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/javascript_test.cljsimilarity index 93%rename from test/metabase/typed_schemas/javascript_test.cljrename to enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/javascript_test.cljindex 85485158701..211fb20fdba 100644--- a/test/metabase/typed_schemas/javascript_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/javascript_test.clj@@ -1,15 +1,15 @@-(ns metabase.typed-schemas.javascript-test+(ns metabase-enterprise.data-apps.generate.schemas.javascript-test(:require[clojure.string :as str][clojure.test :refer :all]- [metabase.typed-schemas.javascript :as javascript]+ [metabase-enterprise.data-apps.generate.schemas.javascript :as schemas.javascript][metabase.util.malli.registry :as mr]))(defn- render-lines"Renders a one-statement module and returns its lines, without the trailingnewline, so goldens read naturally as vectors of lines."[statement]- (str/split-lines (javascript/render-js [:module statement])))+ (str/split-lines (schemas.javascript/render-js [:module statement])))(deftest literal-expressions-test(are [expr rendered] (= [(str "const x = " rendered " as const;")]@@ -97,7 +97,7 @@(deftest metadata-cannot-end-its-comment-early-test(testing "`*/` inside a value is written as `*\\/`, which JSON reads back as `*/`"- (let [rendered (javascript/render-js+ (let [rendered (schemas.javascript/render-js[:module [:const "x" [:obj ["orders" {:metadata {"description" "Paid orders /* see note */"}}[:obj ["id" [:lit 1]]]]]]])](is (str/includes? rendered "\"description\": \"Paid orders /* see note *\\/\""))@@ -120,14 +120,14 @@"const schema = { } as const;\n""\n""export default schema;\n")- (javascript/render-js+ (schemas.javascript/render-js[:module[:raw "function helper() {}"][:const "schema" [:obj]][:export-default [:ref "schema"]]]))))(deftest module-schema-accepts-the-grammar-test- (is (mr/validate javascript/Module+ (is (mr/validate schemas.javascript/Module[:module[:raw "function helper() {}"][:const "tables"@@ -137,7 +137,7 @@[:export-default [:ref "schema"]]])))(deftest module-schema-rejects-unknown-nodes-test- (are [module] (not (mr/validate javascript/Module module))+ (are [module] (not (mr/validate schemas.javascript/Module module))[:module [:const "x" [:string "not-a-node"]]][:module [:const "x" [:obj ["key" {:metadataz {"typo" 1}} [:lit 1]]]]];; Comments were replaced by metadata blocks.@@ -150,6 +150,6 @@(deftest metadata-inside-a-call-argument-is-refused-test(testing "a `:call` argument prints on one line, where a block can't go, so its metadata is an error, not dropped"(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Metadata can only be attached to an object"- (javascript/render-js+ (schemas.javascript/render-js[:module [:const "x" [:call "pickFields"[:obj ["orders" {:metadata {"entityId" "abc"}} [:obj]]]]]])))))diff --git a/test/metabase/typed_schemas/schema/metric_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/metric_test.cljsimilarity index 81%rename from test/metabase/typed_schemas/schema/metric_test.cljrename to enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/metric_test.cljindex 583ccfcf2df..2ada0b524af 100644--- a/test/metabase/typed_schemas/schema/metric_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/metric_test.clj@@ -1,14 +1,13 @@-(ns metabase.typed-schemas.schema.metric-test+(ns metabase-enterprise.data-apps.generate.schemas.metric-test(:require[clojure.test :refer :all]+ [metabase-enterprise.data-apps.db :as data-apps.db]+ [metabase-enterprise.data-apps.generate.schemas.metric :as schemas.metric]+ [metabase-enterprise.data-apps.generate.schemas.table :as schemas.table][metabase.lib.core :as lib][metabase.metabot.core :as metabot][metabase.test :as mt]- [metabase.test.fixtures :as fixtures]- [metabase.typed-schemas.db :as typed-schemas.db]- [metabase.typed-schemas.schema.common :as schema.common]- [metabase.typed-schemas.schema.metric :as schema.metric]- [metabase.typed-schemas.schema.table :as schema.table]))+ [metabase.test.fixtures :as fixtures]))(use-fixtures :once (fixtures/initialize :db))@@ -23,7 +22,7 @@:fieldId 3815:tableId 12:metricId 247}- (#'schema.metric/dimension-schema+ (#'schemas.metric/dimension-schema{:id "550e8400-e29b-41d4-a716-446655440001":name "category":display-name "Category"@@ -35,7 +34,7 @@(deftest ^:parallel metric-dimension-schema-preserves-source-field-id-test(is (= 102(:sourceFieldId- (#'schema.metric/dimension-schema+ (#'schemas.metric/dimension-schema{:id "category-dimension":name "category":source-field-id 102}@@ -44,14 +43,14 @@(deftest ^:parallel metric-source-id-test(testing "integer source-table emits sourceTableId but not sourceCardId"(let [card {:dataset_query {:query {:source-table 10}}}]- (is (= 10 (#'schema.metric/source-table-id card)))- (is (nil? (#'schema.metric/source-card-id card)))))+ (is (= 10 (#'schemas.metric/source-table-id card)))+ (is (nil? (#'schemas.metric/source-card-id card)))))(testing "card source-table emits sourceCardId but not sourceTableId"(let [card {:dataset_query {:query {:source-table "card__42"}}}]- (is (nil? (#'schema.metric/source-table-id card)))- (is (= 42 (#'schema.metric/source-card-id card)))))+ (is (nil? (#'schemas.metric/source-table-id card)))+ (is (= 42 (#'schemas.metric/source-card-id card)))))(testing "stage source-card emits sourceCardId"- (is (= 42 (#'schema.metric/source-card-id+ (is (= 42 (#'schemas.metric/source-card-id{:dataset_query {:stages [{:source-card 42}]}})))))(defn- orders-metric-query@@ -76,14 +75,14 @@:source-field-id (mt/id :orders :product_id)}{:type :literal :value "Widget"}]}])}](is (= ["Created At is greater than or equal to \"2025-01-01\"" "Category is Widget"]- (#'schema.metric/metric-filters card)))))+ (#'schemas.metric/metric-filters card)))))(testing "a metric without filters has none"(mt/with-temp [:model/Card card {:type :metric:database_id (mt/id):dataset_query (orders-metric-query [])}]- (is (nil? (#'schema.metric/metric-filters card)))))+ (is (nil? (#'schemas.metric/metric-filters card)))))(testing "a card without a query has none"- (is (nil? (#'schema.metric/metric-filters {:id 247 :dataset_query {}})))))+ (is (nil? (#'schemas.metric/metric-filters {:id 247 :dataset_query {}})))))(deftest metric-details-skips-default-temporal-breakout-test(let [requested (atom nil)]@@ -92,7 +91,7 @@(reset! requested options){:structured-output {:id 247}})](is (= {:id 247}- (#'schema.metric/metric-details {:id 247})))+ (#'schemas.metric/metric-details {:id 247})))(is (=? {:with-default-temporal-breakout? false}@requested)))))@@ -101,9 +100,9 @@(constantly {:output "Not found.":status-code 404})](let [exception (is (thrown? clojure.lang.ExceptionInfo- (#'schema.metric/metric-details {:id 247- :name "Customer Lifetime Value"- :type :metric})))]+ (#'schemas.metric/metric-details {:id 247+ :name "Customer Lifetime Value"+ :type :metric})))](is (=? {:card-id 247:card-name "Customer Lifetime Value":card-type :metric@@ -112,7 +111,7 @@(ex-data exception))))))(deftest metric-schemas-excludes-card-sourced-metrics-test- (mt/with-dynamic-fn-redefs [schema.common/select-schema-cards+ (mt/with-dynamic-fn-redefs [data-apps.db/metric-cards-in-collections(constantly [{:id 247:dataset_query {:lib/type :mbql/query:database 1@@ -122,13 +121,13 @@:dataset_query {:query {:source-table "card__42"}}}{:id 259:dataset_query {:stages [{:source-card 42}]}}])- schema.metric/metric-details identity- schema.metric/metric-schema (fn [details _card] (:id details))]+ schemas.metric/metric-details identity+ schemas.metric/metric-schema (fn [details _card] (:id details))](is (= [247]- (vec (schema.metric/metric-schemas nil nil))))))+ (vec (schemas.metric/metric-schemas #{1}))))))(deftest metric-schemas-excludes-metrics-that-reference-other-metrics-test- (mt/with-dynamic-fn-redefs [schema.common/select-schema-cards+ (mt/with-dynamic-fn-redefs [data-apps.db/metric-cards-in-collections(constantly [{:id 247:dataset_query {:lib/type :mbql/query:database 1@@ -143,16 +142,16 @@{:lib/uuid"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}247]]}]}}])- schema.metric/metric-details identity- schema.metric/metric-schema (fn [details _card] (:id details))]+ schemas.metric/metric-details identity+ schemas.metric/metric-schema (fn [details _card] (:id details))](is (= [247]- (vec (schema.metric/metric-schemas nil nil))))))+ (vec (schemas.metric/metric-schemas #{1}))))))(deftest metric-schemas-excludes-metrics-that-join-a-saved-question-test;; Metric 258 is table-sourced and joins a saved question, so `source-card-id` — which only reads;; stage 0's source — passes it. A data app's copy of it would reference a saved question outside;; the app's resources, which the pull refuses, so codegen has to drop it here as well.- (mt/with-dynamic-fn-redefs [schema.common/select-schema-cards+ (mt/with-dynamic-fn-redefs [data-apps.db/metric-cards-in-collections(constantly [{:id 247:dataset_query {:lib/type :mbql/query:database 1@@ -172,16 +171,16 @@{:lib/uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}[:field {:lib/uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"} 11][:field {:lib/uuid "cccccccc-cccc-cccc-cccc-cccccccccccc"} 12]]]}]}]}}])- schema.metric/metric-details identity- schema.metric/metric-schema (fn [details _card] (:id details))]+ schemas.metric/metric-details identity+ schemas.metric/metric-schema (fn [details _card] (:id details))](is (= [247]- (vec (schema.metric/metric-schemas nil nil))))))+ (vec (schemas.metric/metric-schemas #{1}))))))(deftest metric-schema-keys-dimensions-test- (mt/with-dynamic-fn-redefs [schema.metric/metric-result-column (constantly nil)- schema.metric/table-source-rows+ (mt/with-dynamic-fn-redefs [schemas.metric/metric-result-column (constantly nil)+ schemas.metric/table-source-rows(constantly [{:id 10 :name "orders" :display_name "Orders"}])- schema.metric/sync-and-fetch-metric-dimensions!+ schemas.metric/sync-and-fetch-metric-dimensions!(constantly [{:id "550e8400-e29b-41d4-a716-446655440001":name "orders":display-name "Orders"@@ -208,23 +207,23 @@:fieldId 42:tableId 10:metricId 247}}}- (#'schema.metric/metric-schema+ (#'schemas.metric/metric-schema{:id 247:name "Customer Lifetime Value"}{:id 247})))))(deftest metric-schema-reuses-table-source-rows-test(let [table-select-count (atom 0)]- (mt/with-dynamic-fn-redefs [schema.metric/metric-result-column (constantly nil)- schema.metric/sync-and-fetch-metric-dimensions!+ (mt/with-dynamic-fn-redefs [schemas.metric/metric-result-column (constantly nil)+ schemas.metric/sync-and-fetch-metric-dimensions!(constantly [{:id "orders-dimension":name "orders":table-id 10}])- typed-schemas.db/table-names+ data-apps.db/table-names(fn [_table-ids](swap! table-select-count inc)[{:id 10 :name "orders" :display_name "Orders"}])]- (#'schema.metric/metric-schema+ (#'schemas.metric/metric-schema{:id 247:name "Customer Lifetime Value"}{:id 247})@@ -237,23 +236,23 @@:sources [{:type :field, :field-id 42}]}{:id "people-dimension":sources [{:type :field, :field-id 84}]}]]- (mt/with-dynamic-fn-redefs [schema.metric/sync-and-fetch-metric-dimensions! (constantly dimensions)- schema.table/table-by-field-id (fn [field-id]- (swap! field-lookup-attempts conj field-id)- ({42 10, 84 20} field-id))- typed-schemas.db/field-ids-and-table-ids+ (mt/with-dynamic-fn-redefs [schemas.metric/sync-and-fetch-metric-dimensions! (constantly dimensions)+ schemas.table/table-by-field-id (fn [field-id]+ (swap! field-lookup-attempts conj field-id)+ ({42 10, 84 20} field-id))+ data-apps.db/field-ids-and-table-ids(fn [_field-ids](swap! field-select-count inc)[{:id 42 :table_id 10}{:id 84 :table_id 20}])](is (= (mapv vector dimensions [10 20])- (#'schema.metric/metric-dimensions-with-table-ids {:id 247} nil)))+ (#'schemas.metric/metric-dimensions-with-table-ids {:id 247} nil)))(is (= 1 @field-select-count))(is (empty? @field-lookup-attempts)))))(deftest source-card-metric-schema-omits-mapped-table-dimensions-test- (mt/with-dynamic-fn-redefs [schema.metric/metric-result-column (constantly nil)- schema.metric/sync-and-fetch-metric-dimensions!+ (mt/with-dynamic-fn-redefs [schemas.metric/metric-result-column (constantly nil)+ schemas.metric/sync-and-fetch-metric-dimensions!(constantly [{:id "count-dimension-uuid":name "count"}{:id "store-name-dimension-uuid"@@ -275,7 +274,7 @@:key "count":id "count-dimension-uuid":metricId 259}}}- (#'schema.metric/metric-schema+ (#'schemas.metric/metric-schema{:id 259:name "Stores with Over 5 Employees"}{:id 259diff --git a/test/metabase/typed_schemas/render_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/render_test.cljsimilarity index 89%rename from test/metabase/typed_schemas/render_test.cljrename to enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/render_test.cljindex 29dbf330da9..da920315e7a 100644--- a/test/metabase/typed_schemas/render_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/render_test.clj@@ -1,16 +1,12 @@-(ns metabase.typed-schemas.render-test+(ns metabase-enterprise.data-apps.generate.schemas.render-test(:require[clojure.string :as str][clojure.test :refer :all]- [metabase.typed-schemas.core :as typed-schemas]- [metabase.typed-schemas.javascript :as javascript]- [metabase.typed-schemas.render :as render]+ [metabase-enterprise.data-apps.generate.schemas :as schemas]+ [metabase-enterprise.data-apps.generate.schemas.javascript :as schemas.javascript]+ [metabase-enterprise.data-apps.generate.schemas.render :as schemas.render][metabase.util.malli.registry :as mr]))-(def ^:private orders-question- {:type "card"- :description "Saved orders"})-(def ^:private orders-table{:type "table":id 10@@ -59,7 +55,6 @@(def ^:private compacting-schema{:schemaVersion 2- :questions {"ordersQuestion" orders-question}:tables {"orders" orders-table"franchises" franchises-table}:metrics {"revenue" revenue-metric}})@@ -75,18 +70,18 @@:jsType "Date"}}}}})(deftest typescript-renderer-emits-metadata-blocks-and-runtime-data-test- (let [body (typed-schemas/render-typescript compacting-schema)]+ (let [body (schemas/render-typescript compacting-schema)](testing "context for agents is a metadata block, never a line comment"(is (not (re-find #"(?m)^\s*//" body)))(testing "an entry opens with its block, so a reader meets the context before the data"- (is (re-find #"(?s)ordersQuestion: \{\n\s*/\* metadata: \{ \"description\": \"Saved orders\" \} \*/\n\s*type: \"card\"" body))+ (is (re-find #"(?s)revenue: \{\n\s*/\* metadata: \{.*?\} \*/\n\s*type: \"metric\"" body))(is (str/includes? body "\"description\": \"Total order revenue\""))(is (str/includes? body (str "/* metadata: { \"displayName\": \"Payment Method\", ""\"semanticType\": \"type/Category\" } */\n type: \"column\""))))(testing "a block with a nested value prints one array item per line"(is (re-find #"(?s)\"filters\": \[\n\s*\"Status is paid\",\n\s*\"Created At is in the previous 30 days\"\n\s*\]" body))))(testing "data for the Lib.createTestQuery DSL stays runtime"- (is (str/includes? body "ordersQuestion: {\n /* metadata:"))+ (is (str/includes? body "revenue: {\n /* metadata:"))(is (str/includes? body "paymentMethod: {\n /* metadata:"))(is (str/includes? body "databaseId: 1"))(is (str/includes? body "sourceTableId: 10"))@@ -96,7 +91,7 @@(is (not (str/includes? body "filters:"))))))(deftest typescript-renderer-omits-what-a-metadata-block-has-nothing-to-say-test- (let [body (typed-schemas/render-typescript+ (let [body (schemas/render-typescript{:schemaVersion 2:tables {"orders" {:type "table":id 10@@ -121,7 +116,7 @@(is (not (str/includes? body "description"))))))(deftest typescript-renderer-compacts-metric-dimensions-test- (let [body (typed-schemas/render-typescript compacting-schema)]+ (let [body (schemas/render-typescript compacting-schema)];; Metric dimensions should compact into pickFields(...) references.(is (str/includes? body "function pickFields"))(is (str/includes? body "const field = fields[key] as { tableId?: number };"))@@ -152,12 +147,12 @@(rest obj-node)))(deftest schema->ast-produces-valid-modules-test- (are [schema] (mr/validate javascript/Module (render/schema->ast schema))+ (are [schema] (mr/validate schemas.javascript/Module (schemas.render/schema->ast schema))compacting-schemaraw-dimensions-schema))(deftest schema->ast-compacts-metric-dimensions-test- (let [ast (render/schema->ast compacting-schema)+ (let [ast (schemas.render/schema->ast compacting-schema)dimensions (-> (module-const ast "metrics")(obj-entry "revenue")(obj-entry :dimensions))]@@ -172,7 +167,7 @@(obj-entry dimensions "franchises")))))(deftest schema->ast-splits-runtime-keys-from-metadata-test- (let [ast (render/schema->ast compacting-schema)+ (let [ast (schemas.render/schema->ast compacting-schema)fields (-> (module-const ast "tables")(obj-entry "orders")(obj-entry :fields))@@ -188,7 +183,7 @@(is (nil? (obj-entry field-node :displayName))))))(deftest typescript-renderer-omits-pick-fields-helper-for-raw-dimensions-test- (let [body (typed-schemas/render-typescript raw-dimensions-schema)]+ (let [body (schemas/render-typescript raw-dimensions-schema)];; Dimensions that cannot be resolved to table fields stay as raw fields, so;; the rendered module should not include the pickFields helper.(is (not (str/includes? body "function pickFields")))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/source_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/source_test.cljnew file mode 100644index 00000000000..939104b350d--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/source_test.clj@@ -0,0 +1,134 @@+(ns metabase-enterprise.data-apps.generate.schemas.source-test+ "Tests [[schemas.source/app-db-source]] against real application-database rows.++ Everything downstream of the source works on plain data and is covered by+ cheap in-memory tests; these tests are the seam where the data app schema meets the+ database."+ (:require+ [clojure.set :as set]+ [clojure.test :refer :all]+ [metabase-enterprise.data-apps.generate.schemas.source :as schemas.source]+ [metabase-enterprise.data-apps.test-util :as data-apps.tu]+ [metabase.collections.models.collection :as collection]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.permissions.core :as perms]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [metabase.util.malli.registry :as mr]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db :test-users))++(deftest library-scope-and-library-tables-test+ (data-apps.tu/do-with-library!+ (fn [{:keys [data-id metrics-id]}]+ (mt/with-temp [:model/Collection data-child {:name "Data child"+ :type "library-data"+ :location (collection/children-location (t2/select-one :model/Collection :id data-id))}+ :model/Collection plain-collection {:name "Not A Library"}+ :model/Database db {}+ :model/Table published {:db_id (:id db), :name "published_table", :active true+ :is_published true, :collection_id data-id}+ :model/Table nested {:db_id (:id db), :name "nested_table", :active true+ :is_published true, :collection_id (:id data-child)}+ :model/Table _unpublished {:db_id (:id db), :name "unpublished_table", :active true+ :is_published false, :collection_id data-id}+ :model/Table _elsewhere {:db_id (:id db), :name "elsewhere_table", :active true+ :is_published true, :collection_id (:id plain-collection)}]+ (mt/with-current-user (mt/user->id :crowberto)+ (let [scope (schemas.source/library-scope schemas.source/app-db-source)]+ (testing "library scope classifies the root libraries' collections by type"+ (is (mr/validate schemas.source/LibraryScope scope))+ (is (= {:data-collection-ids #{data-id (:id data-child)}+ :metric-collection-ids #{metrics-id}}+ scope)))+ (testing "library tables are the published tables in the data collections and their descendants"+ (is (= #{(:id published) (:id nested)}+ (into #{}+ (map :id)+ (schemas.source/library-tables schemas.source/app-db-source (:data-collection-ids scope)))))+ (is (= [] (schemas.source/library-tables schemas.source/app-db-source #{}))))))))))++(deftest metrics-honour-collection-ids-test+ (data-apps.tu/do-with-library!+ (fn [{:keys [metrics-id]}]+ (mt/with-temp [:model/Collection child {:name "Child"+ :type "library-metrics"+ :location (collection/children-location (t2/select-one :model/Collection :id metrics-id))}+ :model/Card _metric {:name "Library revenue"+ :type :metric+ :collection_id (:id child)}]+ (mt/with-test-user :crowberto+ (let [{:keys [metric-collection-ids]} (schemas.source/library-scope schemas.source/app-db-source)]+ (is (some #{"libraryRevenue"}+ (map :key (schemas.source/metrics schemas.source/app-db-source metric-collection-ids))))+ (is (= [] (schemas.source/metrics schemas.source/app-db-source #{})))))))))++(deftest tables-test+ (mt/with-temp [:model/Database db {}+ :model/Table table {:db_id (:id db), :name "widgets", :display_name "Widgets", :active true}+ :model/Field _ {:table_id (:id table), :name "price", :base_type :type/Float}]+ (mt/with-current-user (mt/user->id :crowberto)+ (testing "returns shaped table entities for the table ids"+ (is (=? [{:type "table"+ :key "widgets"+ :fields {"price" {:jsType "number"}}}]+ (schemas.source/tables schemas.source/app-db-source #{(:id table)}))))+ (testing "an empty table-id set matches nothing"+ (is (= [] (schemas.source/tables schemas.source/app-db-source #{})))))))++(defn- touch-category-query []+ (let [query (lib/native-query (mt/metadata-provider) "UPDATE categories SET name = name WHERE id = {{id}}")]+ (lib/with-template-tags query (mapv #(assoc % :type :number) (lib/template-tags query)))))++(def ^:private id-parameter+ {:id "id", :slug "id", :type :number/=, :target [:variable [:template-tag "id"]], :required true})++(deftest actions-test+ (mt/with-temp [:model/Collection {collection-id :id} {:name "Actions" :namespace "data-actions"}+ :model/Collection {app-collection-id :id} {:name "Data App: orders" :namespace "data-apps"}+ :model/Card {model-id :id} {:type :model+ :dataset_query (lib/query (mt/metadata-provider)+ (lib.metadata/table (mt/metadata-provider)+ (mt/id :categories)))}+ :model/Action {standalone :id} {:type :query, :name "Touch category", :parameters [id-parameter]}+ :model/QueryAction _ {:action_id standalone, :dataset_query (touch-category-query)}+ :model/Action {archived :id} {:type :query, :name "Archived", :archived true}+ :model/QueryAction _ {:action_id archived, :dataset_query (touch-category-query)}+ :model/Action {in-collection :id} {:type :query, :name "In a collection", :collection_id collection-id}+ :model/QueryAction _ {:action_id in-collection, :dataset_query (touch-category-query)}+ :model/Action {on-model :id} {:type :query, :name "On a model", :model_id model-id}+ :model/QueryAction _ {:action_id on-model, :dataset_query (touch-category-query)}+ :model/Action {app-copy :id} {:type :query, :name "App copy", :collection_id app-collection-id}+ :model/QueryAction _ {:action_id app-copy, :dataset_query (touch-category-query)}]+ (let [action-ids (fn [] (into #{} (map :id) (schemas.source/actions schemas.source/app-db-source)))+ ours #{standalone archived in-collection on-model app-copy}]+ (mt/with-current-user (mt/user->id :crowberto)+ (testing "returns the unarchived actions without a model"+ (is (= #{standalone in-collection app-copy} (set/intersection ours (action-ids)))))+ (testing "leaves out the copies a data app owns"+ (mt/with-dynamic-fn-redefs [perms/data-app-collection-ids (constantly #{app-collection-id})]+ (is (= #{standalone in-collection} (set/intersection ours (action-ids))))))+ (testing "an action renders with its parameters typed from the template tag"+ (is (=? [{:kind "action"+ :key "touchCategory"+ :id standalone+ :type "query"+ :parameters [{:slug "id", :jsType "number", :required true}]}]+ (filter #(= standalone (:id %)) (schemas.source/actions schemas.source/app-db-source)))))))))++(deftest actions-leave-out-those-backed-by-a-routing-destination-test+ (mt/with-temp [:model/Database {router-id :id} {}+ :model/Database {destination-id :id} {:router_database_id router-id}+ :model/Action {open :id} {:type :query, :name "Open action"}+ :model/QueryAction _ {:action_id open, :dataset_query (touch-category-query)}+ :model/Action {routed :id} {:type :query, :name "Routed action"}+ :model/QueryAction _ {:action_id routed+ :dataset_query (lib/native-query (lib-be/application-database-metadata-provider destination-id)+ "UPDATE categories SET name = name")}]+ (mt/with-current-user (mt/user->id :crowberto)+ (let [ids (into #{} (map :id) (schemas.source/actions schemas.source/app-db-source))]+ (is (contains? ids open))+ (is (not (contains? ids routed)))))))diff --git a/test/metabase/typed_schemas/schema/table_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/table_test.cljsimilarity index 85%rename from test/metabase/typed_schemas/schema/table_test.cljrename to enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/table_test.cljindex ab68b93d9ea..120ed9443ed 100644--- a/test/metabase/typed_schemas/schema/table_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/table_test.clj@@ -1,13 +1,13 @@-(ns metabase.typed-schemas.schema.table-test+(ns metabase-enterprise.data-apps.generate.schemas.table-test(:require[clojure.test :refer :all]+ [metabase-enterprise.data-apps.generate.schemas.common :as schemas.common]+ [metabase-enterprise.data-apps.generate.schemas.table :as schemas.table][metabase.audit-app.core :as audit][metabase.config.core :as config][metabase.lib-be.core :as lib-be][metabase.metabot.core :as metabot][metabase.test :as mt]- [metabase.typed-schemas.schema.common :as schema.common]- [metabase.typed-schemas.schema.table :as schema.table][toucan2.core :as t2]))(def ^:private created-at-field@@ -49,7 +49,7 @@(deftest field-schema-uses-field-id-test(is (= created-at-field-schema- (schema.table/field-schema created-at-field))))+ (schemas.table/field-schema created-at-field))))(deftest table-schema-keys-fields-test(is (= {:type "table"@@ -59,7 +59,7 @@:databaseName "Boba":tableName "orders":fields {"createdAt" (assoc created-at-field-schema :sourceName "orders")}}- (schema.table/table-schema orders-table))))+ (schemas.table/table-schema orders-table)))); Avoid N+1 on table fields by re-using the known table id(deftest table-schema-uses-known-table-id-for-detail-fields-test@@ -75,7 +75,7 @@(with-redefs [t2/select-one-fn (fn [& _](swap! field-table-lookups inc)999)]- (let [schema (schema.table/table-schema (assoc orders-table :fields detail-fields))]+ (let [schema (schemas.table/table-schema (assoc orders-table :fields detail-fields))](is (= #{10} (->> (:fields schema) vals (map :tableId) set)))(is (zero? @field-table-lookups))))))@@ -85,7 +85,7 @@:id 12:tableId 10:name "Completed Orders"}- (schema.table/segment-schema+ (schemas.table/segment-schema10{:id 12:name "Completed Orders"@@ -93,7 +93,7 @@(deftest measure-schema-uses-result-column-test(testing "measure result columns come from the measure definition when available"- (mt/with-dynamic-fn-redefs [schema.table/measure-result-column+ (mt/with-dynamic-fn-redefs [schemas.table/measure-result-column(constantly {:name "sum":display_name "Sum of Total":base_type "type/Decimal"})]@@ -103,19 +103,19 @@:displayName "Sum of Total":baseType "type/Decimal":jsType "number"}])- (schema.table/measure-schema 10 2 total-revenue-measure)))))+ (schemas.table/measure-schema 10 2 total-revenue-measure)))))(testing "measure result columns fall back to the measure name"- (mt/with-dynamic-fn-redefs [schema.table/measure-result-column (constantly nil)]+ (mt/with-dynamic-fn-redefs [schemas.table/measure-result-column (constantly nil)](is (= (assoc total-revenue-schema:columns [{:type "column", :name "Total Revenue", :displayName "Total Revenue", :jsType "unknown"}])- (schema.table/measure-schema 10 2 total-revenue-measure))))))+ (schemas.table/measure-schema 10 2 total-revenue-measure))))))(deftest table-schemas-surface-detail-error-responses-test(mt/with-dynamic-fn-redefs [metabot/get-table-details(constantly {:output "Not found.":status-code 404})](let [exception (is (thrown? clojure.lang.ExceptionInfo- (doall (schema.table/table-schemas [{:id 10 :name "Orders"}]))))]+ (doall (schemas.table/table-schemas [{:id 10 :name "Orders"}]))))](is (=? {:table-id 10:table-name "Orders":status-code 404@@ -139,7 +139,7 @@:created_at :%now:updated_at :%now})](mt/with-test-user :crowberto- (let [tables (schema.table/select-tables nil [open-table-id destination-table-id])]+ (let [tables (schemas.table/select-tables #{open-table-id destination-table-id})](is (= [open-table-id] (map :id tables)))))))))(deftest the-audit-databases-tables-are-left-out-without-the-audit-feature-test@@ -147,12 +147,12 @@(testing "the table details lookup refuses the audit database's tables while the feature is off, so the schemaleaves them out rather than fail an unscoped request"(mt/with-premium-features #{}- (is (= [1] (map :id (#'schema.table/without-unavailable-tables tables))))))+ (is (= [1] (map :id (#'schemas.table/without-unavailable-tables tables))))));; a premium feature is never on without the EE code, whatever the token says(when config/ee-available?(testing "with the feature on they are listed"(mt/with-premium-features #{:audit-app}- (is (= [1 2] (map :id (#'schema.table/without-unavailable-tables tables)))))))))+ (is (= [1 2] (map :id (#'schemas.table/without-unavailable-tables tables)))))))));; Batch measure definitions to avoid N+1 queries.(deftest table-schema-bulk-loads-measure-definitions-test@@ -163,13 +163,13 @@(with-redefs [lib-be/application-database-metadata-provider (fn [_database-id](swap! metadata-provider-count inc):metadata-provider)- schema.common/aggregation-result-column-with-metadata-provider (constantly nil)+ schemas.common/aggregation-result-column-with-metadata-provider (constantly nil)t2/select (fn [columns & _args](when (= columns [:model/Measure :id :definition])(swap! measure-select-count inc)[{:id 1 :definition [:aggregation 1]}{:id 2 :definition [:aggregation 2]}]))]- (schema.table/table-schema (assoc orders-table :measures measures))+ (schemas.table/table-schema (assoc orders-table :measures measures))(is (= 1 @measure-select-count))(is (= 1 @metadata-provider-count)))))@@ -187,5 +187,5 @@:weekday-distribution [0.1 0.1 0.1 0.1 0.2 0.2 0.2]:hour-distribution (vec (repeat 24 (/ 1.0 24)))}}}}](mt/with-test-user :crowberto- (let [schema (first (schema.table/table-schemas [table]))]+ (let [schema (first (schemas.table/table-schemas [table]))](is (= #{"customerId" "lastOrderedAt"} (set (keys (:fields schema)))))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_api_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_api_test.cljnew file mode 100644index 00000000000..b8ae8d965d3--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_api_test.clj@@ -0,0 +1,84 @@+(ns metabase-enterprise.data-apps.generate.schemas-api-test+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.data-apps.test-util :as data-apps.tu]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.permissions.core :as perms]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db :web-server :test-users))++(defn- get-schemas+ "The response of `GET /api/apps/generate/schemas` as `user`, expecting `status`, with the data apps feature on."+ [user status]+ (mt/with-premium-features #{:data-apps}+ (mt/user-http-request-full-response user :get status "apps/generate/schemas")))++(deftest only-a-superuser-generates-the-schemas-test+ (testing "the schema is what an app's author builds from, and only an admin works with an app's repository"+ (mt/with-premium-features #{:data-apps}+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :get 403 "apps/generate/schemas")))+ (is (string? (mt/user-http-request :crowberto :get 200 "apps/generate/schemas"))))))++(deftest typescript-endpoint-test+ (let [response (get-schemas :crowberto 200)]+ (is (= "text/typescript; charset=utf-8" (get-in response [:headers "Content-Type"])))+ (is (str/includes? (:body response) "\nconst schema = {"))+ (is (str/includes? (:body response) "\n schemaVersion: 2"))+ (is (not (str/includes? (:body response) "\n questions:")))+ (is (str/includes? (:body response) "\n tables: tables"))+ (is (str/includes? (:body response) "\n metrics: metrics"))+ (is (str/ends-with? (:body response) "export default schema;\n"))+ (is (not (str/includes? (:body response) "\"schemaVersion\"")))+ (is (not (str/includes? (:body response) "operators: [ ]")))+ (is (not (str/includes? (:body response) "parameters: [ ]")))+ (is (not (str/includes? (:body response) "verified: false")))))++(deftest schemas-cover-the-libraries-and-the-model-less-actions-test+ (mt/dataset test-data+ (mt/with-temp-copy-of-db+ (mt/with-actions-enabled+ (data-apps.tu/do-with-library!+ (fn [{:keys [data-id metrics-id]}]+ (let [mp (mt/metadata-provider)+ orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ revenue-query (lib/aggregate orders-query+ (lib/sum (lib.metadata/field mp (mt/id :orders :total))))]+ (t2/update! :model/Table (mt/id :orders) {:is_published true, :collection_id data-id})+ (mt/with-temp [:model/Card _metric {:name "Order revenue", :database_id (mt/id), :table_id (mt/id :orders)+ :type :metric, :display :scalar, :collection_id metrics-id+ :dataset_query revenue-query}+ :model/Card model {:name "Order model", :database_id (mt/id), :table_id (mt/id :orders)+ :type :model+ :dataset_query orders-query+ :result_metadata [{:name "total", :display_name "Total"+ :base_type :type/Float+ :field_ref [:field (mt/id :orders :total) nil]+ :id (mt/id :orders :total)}]}+ :model/Action action {:name "Update order", :model_id (:id model), :type :implicit}+ :model/ImplicitAction _ {:action_id (:id action), :kind "row/update"}+ :model/Action standalone {:name "Discount order", :type :query}+ :model/QueryAction _ {:action_id (:id standalone)+ :dataset_query (lib/native-query mp "UPDATE orders SET discount = 0")}+ :model/Collection copies {:name "Data App: orders", :namespace "data-apps"}+ :model/Action copy {:name "Copied order", :type :query, :collection_id (:id copies)}+ :model/QueryAction _ {:action_id (:id copy)+ :dataset_query (lib/native-query mp "UPDATE orders SET discount = 0")}]+ (let [body (:body (get-schemas :crowberto 200))]+ (testing "a table in the data library is listed"+ (is (str/includes? body " orders: {"))+ (is (str/includes? body "name: \"Orders\"")))+ (testing "a metric in the metrics library is listed"+ (is (str/includes? body "name: \"Order revenue\"")))+ (testing "an action without a model is listed, and one with a model never is"+ (is (str/includes? body "discountOrder"))+ (is (not (str/includes? body "updateOrder"))))+ (testing "a copy a data app owns stays out"+ (is (str/includes? body "copiedOrder"))+ (mt/with-dynamic-fn-redefs [perms/data-app-collection-ids (constantly #{(:id copies)})]+ (is (not (str/includes? (:body (get-schemas :crowberto 200)) "copiedOrder"))))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_test.cljnew file mode 100644index 00000000000..62b3cc244a0--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_test.clj@@ -0,0 +1,144 @@+(ns metabase-enterprise.data-apps.generate.schemas-test+ "Data app schema tests form a pyramid; put new tests at the lowest level that+ can express them:++ - In-memory (no db): printer goldens in `javascript-test` (the output format+ spec), structural =? AST tests in `render-test`, pure shaping in+ `common-test` and the `schemas.*` tests, assembly from literal [[Items]]+ against a reified literal source here.+ - Db seam (with-temp): `source-test` for `app-db-source`.+ - One end-to-end test here on `mt/dataset test-data`, through the whole+ pipeline to rendered TypeScript."+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.data-apps.generate.schemas :as schemas]+ [metabase-enterprise.data-apps.generate.schemas.source :as schemas.source]+ [metabase-enterprise.data-apps.test-util :as data-apps.tu]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db :test-users))++(def ^:private test-info+ {:generated-at "2026-01-01T00:00:00Z"+ :instance-url "https://metabase.example.com"})++(deftest create-schema-assembles-items-test+ (is (= {:schemaVersion 2+ :generatedAt "2026-01-01T00:00:00Z"+ :metabase {:instanceUrl "https://metabase.example.com"}+ :actions {"shipOrder" {:kind "action", :key "shipOrder", :id 11}}+ :tables {"orders" {:type "table", :key "orders", :id 3}}+ :metrics {"revenue" {:type "metric", :key "revenue", :id 2}}}+ (schemas/create-schema+ {:actions [{:kind "action", :key "shipOrder", :id 11}]+ :tables [{:type "table", :key "orders", :id 3}]+ :metrics [{:type "metric", :key "revenue", :id 2}]}+ test-info))))++(deftest create-schema-disambiguates-duplicate-keys-test+ (let [schema (schemas/create-schema+ {:actions []+ :tables [{:type "table", :key "orders", :id 3}+ {:type "table", :key "orders", :id 4}]+ :metrics []}+ test-info)]+ (is (= ["orders3" "orders4"] (keys (:tables schema))))+ (is (= ["orders3" "orders4"] (map :key (vals (:tables schema)))))))++(deftest create-schema-defaults-info-test+ (let [schema (schemas/create-schema {:actions [], :tables [], :metrics []})]+ (is (string? (:generatedAt schema)))+ (is (contains? (:metabase schema) :instanceUrl))))++(defn- literal-source+ "A [[schemas.source/SchemaSource]] over literal values. `tables` are filtered by the+ requested table ids so tests can see which tables fetching asked for."+ [{:keys [library-scope library-tables actions metrics tables]}]+ (reify schemas.source/SchemaSource+ (library-scope [_] library-scope)+ (actions [_] (vec actions))+ (metrics [_ _] (vec metrics))+ (tables [_ table-ids] (filterv #(contains? table-ids (:id %)) tables))+ (library-tables [_ _] (vec library-tables))))++(deftest fetch-items-includes-tables-mapped-by-library-metrics-test+ (let [source (literal-source+ {:library-scope {:metric-collection-ids #{20}+ :data-collection-ids #{10}}+ :library-tables [{:id 10}]+ :actions [{:kind "action", :key "shipOrder", :id 11}]+ :metrics [{:type "metric", :key "revenue", :id 1, :mappedTableIds [42]}]+ :tables [{:id 10, :type "table", :key "publishedTable"}+ {:id 42, :type "table", :key "mappedTable"}+ {:id 99, :type "table", :key "notInScope"}]})]+ (is (= {:actions [{:kind "action", :key "shipOrder", :id 11}]+ :tables [{:id 10, :type "table", :key "publishedTable"}+ {:id 42, :type "table", :key "mappedTable"}]+ :metrics [{:type "metric", :key "revenue", :id 1, :mappedTableIds [42]}]}+ (schemas/fetch-items source)))))++(deftest fetch-items-hands-the-source-the-library-scope-test+ (let [metric-collections (atom nil)+ library-tables (atom nil)+ source (reify schemas.source/SchemaSource+ (library-scope [_] {:data-collection-ids #{10} :metric-collection-ids #{20}})+ (actions [_] [])+ (metrics [_ collection-ids] (reset! metric-collections collection-ids) [])+ (tables [_ _] [])+ (library-tables [_ collection-ids] (reset! library-tables collection-ids) []))]+ (schemas/fetch-items source)+ (is (= #{20} @metric-collections))+ (is (= #{10} @library-tables))))++(deftest full-pipeline-end-to-end-test+ (mt/dataset test-data+ (mt/with-temp-copy-of-db+ (mt/with-actions-enabled+ (data-apps.tu/do-with-library!+ (fn [{:keys [data-id metrics-id]}]+ (let [mp (mt/metadata-provider)+ orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ revenue-query (lib/aggregate orders-query+ (lib/sum (lib.metadata/field mp (mt/id :orders :total))))]+ (t2/update! :model/Table (mt/id :orders) {:is_published true, :collection_id data-id})+ (mt/with-temp [:model/Card _metric {:name "Order revenue", :database_id (mt/id), :table_id (mt/id :orders)+ :type :metric, :display :scalar, :collection_id metrics-id+ :dataset_query revenue-query}+ :model/Card model {:name "Order model", :database_id (mt/id), :table_id (mt/id :orders)+ :type :model+ :dataset_query orders-query+ :result_metadata [{:name "total", :display_name "Total"+ :base_type :type/Float+ :field_ref [:field (mt/id :orders :total) nil]+ :id (mt/id :orders :total)}]}+ :model/Action action {:name "Update order", :model_id (:id model), :type :implicit}+ :model/ImplicitAction _ {:action_id (:id action), :kind "row/update"}+ :model/Action standalone {:name "Discount order", :type :query}+ :model/QueryAction _ {:action_id (:id standalone)+ :dataset_query (lib/native-query mp "UPDATE orders SET discount = 0 WHERE id = {{id}}")}]+ (mt/with-current-user (mt/user->id :crowberto)+ (let [schema (schemas/create-schema (schemas/fetch-items) test-info)+ body (schemas/render-typescript schema)]+ (testing "the library's table and metric and the model-less action land in the schema with their real relationships"+ (is (=? {:generatedAt "2026-01-01T00:00:00Z"+ :metabase {:instanceUrl "https://metabase.example.com"}+ :tables {"orders" {:fields {"total" {:jsType "number"}}}}+ :metrics {"orderRevenue" {:mappedTableIds [(mt/id :orders)]+ :columns [{:displayName "Sum of Total"+ :jsType "number"}]}}+ :actions {"discountOrder" {:kind "action", :id (:id standalone), :type "query"}}}+ schema)))+ (testing "saved questions are absent from the schema"+ (is (not (contains? schema :questions))))+ (testing "an action that belongs to a model stays out"+ (is (not (str/includes? body "updateOrder"))))+ (testing "the rendered module carries the real entities"+ (is (str/includes? body "orders: {"))+ (is (str/includes? body "name: \"Order revenue\""))+ (is (str/includes? body "discountOrder: {"))+ (is (str/ends-with? body "export default schema;\n")))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/resource_serialization_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/resource_serialization_test.cljdeleted file mode 100644index a8836315b6a..00000000000--- a/enterprise/backend/test/metabase_enterprise/data_apps/resource_serialization_test.clj+++ /dev/null@@ -1,495 +0,0 @@-(ns metabase-enterprise.data-apps.resource-serialization-test- "`POST /api/apps/serialize-resources`: what the files of a data app's collection are written from."- (:require- [clojure.string :as str]- [clojure.test :refer :all]- [medley.core :as m]- [metabase-enterprise.data-apps.resource-serialization :as resource-serialization]- [metabase-enterprise.data-apps.test-util :as data-apps.tu]- [metabase.actions.core :as actions]- [metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata]- [metabase.models.serialization :as serdes]- [metabase.test :as mt]- [metabase.util :as u]- [metabase.util.malli.fn :as mu.fn]- [toucan2.core :as t2]))--(set! *warn-on-reflection* true)--(def ^:private app-collection "appCollectionEntity01")--(defn- with-ids- "`queries` with an entity ID each, as a definition carries one, unless a query brings its own."- [queries]- (mapv #(merge {:entity_id (u/generate-nano-id)} %) queries))--(defn- serialize!- "Serialize `body` as `user`, in the app collection `app-collection` unless `body` names one, each query with an entity- ID unless it brings one."- ([user status body]- (serialize! user status body #{:data-apps}))- ([user status body features]- (mt/with-premium-features features- (mt/user-http-request user :post status "apps/serialize-resources"- (cond-> (merge {:collection app-collection} body)- (:queries body) (update :queries with-ids))))))--(deftest serialization-needs-each-querys-entity-id-test- (testing "the saved question is written with the definition's entity ID, so every query names one"- (mt/with-premium-features #{:data-apps}- (is (=? {:errors {:queries some?}}- (mt/user-http-request :crowberto :post 400 "apps/serialize-resources"- {:collection app-collection- :queries [{:export "Venues" :query {:stages [{:source {:type "table" :id (mt/id :venues)}}]}}]- :actions []}))))))--(deftest serialization-needs-the-apps-collection-test- (testing "the saved question is written into the app's collection, so the request names it"- (mt/with-premium-features #{:data-apps}- (is (=? {:errors {:collection some?}}- (mt/user-http-request :crowberto :post 400 "apps/serialize-resources" {:queries [] :actions []}))))))--(defn- db-name []- (t2/select-one-fn :name :model/Database (mt/id)))--(defn- table-path [table]- [(db-name) "PUBLIC" table])--(defn- field-path [table field]- [(db-name) "PUBLIC" table field])--(defn- venues-column [field-name & {:as extra}]- (merge {:type "column" :name field-name :tableId (mt/id :venues) :sourceName "VENUES"} extra))--(deftest serializes-a-query-with-every-reference-portable-test- (let [response (serialize! :crowberto 200- {:queries [{:export "PriceByCategory"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}- :filters [{:type "operator" :operator ">"- :args [(venues-column "PRICE")- {:type "literal" :value 1}]}]- :aggregations [{:type "operator" :operator "sum" :name "total"- :args [(venues-column "PRICE")]}]- :breakouts [{:type "column"- :name "NAME"- :sourceName "CATEGORIES"- :sourceFieldId (mt/id :venues :category_id)}]- :orderBys [{:type "column"- :name "total"- :direction "desc"}]- :limit 5}]}}]})]- (testing "the query Metabase builds from the definition, referencing tables and fields by name"- (let [price (field-path "VENUES" "PRICE")- category-id (field-path "VENUES" "CATEGORY_ID")- category (field-path "CATEGORIES" "NAME")]- (is (=? {:queries [{:export "PriceByCategory"- :metrics []- :entity {:dataset_query {:lib/type "mbql/query"- :database (db-name)- :stages [{:source-table (table-path "VENUES")- :filters [[">" {} ["field" {} price] 1]]- :aggregation [["sum" {:name "total"} ["field" {} price]]]- :breakout [["field" {:source-field category-id} category]]- :order-by [["desc" {} ["aggregation"- {:lib/source-name "total"}- string?]]]- :limit 5}]}}}]}- response))))- (testing "only the uuid the order by points at is kept, and it is the aggregation's"- (let [stage (-> response :queries first :entity :dataset_query :stages first)]- (is (= (get-in stage [:aggregation 0 1 :lib/uuid])- (get-in stage [:order-by 0 2 2])))- (is (nil? (get-in stage [:filters 0 1 :lib/uuid])))))))--(deftest only-a-superuser-serializes-test- (testing "an app's resources are written into its repository, which only an admin works with"- (let [body {:queries [{:export "Venues" :query {:stages [{:source {:type "table" :id (mt/id :venues)}}]}}]}]- (is (= "You don't have permissions to do that." (serialize! :rasta 403 body)))- (is (=? {:queries [{:export "Venues" :entity map?}]} (serialize! :crowberto 200 body))))))--(deftest lists-the-metrics-a-query-aggregates-test- (data-apps.tu/do-with-sources!- (fn [{:keys [metric-id]}]- (let [metric-eid (t2/select-one-fn :entity_id :model/Card :id metric-id)]- (is (=? {:queries [{:export "VenueCount"- :metrics [metric-eid]- :entity {:dataset_query {:stages [{:aggregation [["metric" {} metric-eid]]}]}}}]- :metrics [{:id metric-id- :entity {:entity_id metric-eid- :type "metric"- :dataset_query {:stages [{:source-table (table-path "VENUES")}]}}}]}- (serialize! :crowberto 200- {:queries [{:export "VenueCount"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}- :aggregations [{:type "metric" :id metric-id}]}]}}]})))))))--(deftest answers-each-query-on-its-own-test- (is (=? {:queries [{:export "Venues" :entity {:dataset_query {:stages [{:source-table (table-path "VENUES")}]}}}- {:export "Broken" :error "No column found"}]}- (serialize! :crowberto 200- {:queries [{:export "Venues"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}}]}}- {:export "Broken"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}- :fields [(venues-column "NOT_A_COLUMN")]}]}}]}))))--(deftest serializes-a-query-action-test- (data-apps.tu/do-with-sources!- (fn [{:keys [action-id]}]- (let [response (serialize! :crowberto 200 {:actions [action-id]})]- (is (=? {:actions [{:id action-id- :entity {:entity_id (t2/select-one-fn :entity_id :model/Action :id action-id)- :type "query"- :query [{:database_id (db-name)- :dataset_query {:database (db-name)}}]}}]}- response))- (is (not (contains? (-> response :actions first :entity) :model_id))- "it names no model")))))--(deftest refuses-an-action-that-belongs-to-a-model-test- (testing "a data app runs only actions that belong to no model, as the typed schema lists only those"- (data-apps.tu/do-with-sources!- (fn [{:keys [model-action-id]}]- (is (=? {:actions [{:id model-action-id :error #"Action \d+ belongs to a model\..*"}]}- (serialize! :crowberto 200 {:actions [model-action-id]})))))))--(deftest refuses-what-a-copy-cannot-hold-test- (data-apps.tu/do-with-sources!- (fn [{:keys [metric-id model-id]}]- (let [mp (mt/metadata-provider)- model-count (lib/aggregate (lib/query mp (lib.metadata/card mp model-id)) (lib/count))]- (mt/with-temp [:model/Card {reading-metric-id :id} {:name "Metric reading a model"- :type :metric- :database_id (mt/id)- :dataset_query model-count}]- (let [card-sql-id (actions/insert! {:name "Read a card"- :type :query- :database_id (mt/id)- :dataset_query (lib/native-query- mp (str "SELECT * FROM {{#" metric-id "}}"))})- reads-metric {:stages [{:source {:type "table" :id (mt/id :venues)}- :aggregations [{:type "metric" :id reading-metric-id}]}]}- response (serialize! :crowberto 200- {:queries [{:export "ReadsMetric" :query reads-metric}]- :actions [card-sql-id Integer/MAX_VALUE]})]- (testing "a query action whose SQL reads a card"- (is (=? {:id card-sql-id :error (re-pattern (str ".*reads card " metric-id ".*"))}- (nth (:actions response) 0))))- (testing "an action that does not exist"- (is (=? {:error #".*does not exist.*"} (nth (:actions response) 1))))- (testing "a metric that reads a card; the query that aggregates it is still built"- (is (=? {:queries [{:export "ReadsMetric" :entity map?}]- :metrics [{:id reading-metric-id :error (re-pattern (str ".*reads card " model-id ".*"))}]}- response)))))))))--(deftest rejects-unsupported-definitions-test- (testing "the request accepts only what data app definitions support"- (doseq [path [[:queries 0 :query :stages 0 :joins]- [:queries 0 :query :stages 0 :expressions]]]- (testing (pr-str path)- (serialize! :crowberto 400- (assoc-in {:queries [{:export "Venues"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}}]}}]}- path []))))))--(deftest refuses-a-table-that-does-not-exist-test- (is (=? {:queries [{:export "Nothing" :error (str "Table " Integer/MAX_VALUE " does not exist.")}]}- (serialize! :crowberto 200- {:queries [{:export "Nothing"- :query {:stages [{:source {:type "table" :id Integer/MAX_VALUE}}]}}]})))- (testing "a deactivated table is gone too, as it is from the typed schema"- (mt/with-temp [:model/Table {table-id :id} {:db_id (mt/id) :active false}]- (is (=? {:queries [{:export "Gone" :error (str "Table " table-id " does not exist.")}]}- (serialize! :crowberto 200- {:queries [{:export "Gone"- :query {:stages [{:source {:type "table" :id table-id}}]}}]}))))))--(deftest refuses-a-column-that-reaches-a-deactivated-table-test- (testing "types generated before categories was deactivated still select its name through venues.category_id, and- the query would fail on the implicit join when it runs"- (mt/with-temp-vals-in-db :model/Table (mt/id :categories) {:active false}- (is (=? {:queries [{:export "VenueCategories" :error (str "Table " (mt/id :categories) " does not exist.")}]}- (serialize! :crowberto 200- {:queries [{:export "VenueCategories"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}- :fields [{:type "column" :name "NAME" :source-field-id (mt/id :venues :category_id)}]}]}}]}))))))--(deftest refuses-a-metric-that-reaches-a-deactivated-table-test- (testing "a metric the query aggregates reads categories through venues.category_id in its own query, which the- built query holds only the ID of, and the query would fail on the table when it runs"- (let [mp (mt/metadata-provider)- venues (lib/query mp (lib.metadata/table mp (mt/id :venues)))- categories (mt/id :categories)- category (m/find-first (comp #{(mt/id :categories :name)} :id) (lib/filterable-columns venues))]- (mt/with-temp [:model/Card {bars-id :id} {:name "Bars"- :type :metric- :database_id (mt/id)- :dataset_query (-> venues- (lib/filter (lib/= category "Bar"))- (lib/aggregate (lib/count)))}- :model/Card {count-id :id} {:name "Venue count"- :type :metric- :database_id (mt/id)- :dataset_query (lib/aggregate venues (lib/count))}]- (mt/with-temp-vals-in-db :model/Table categories {:active false}- (is (=? {:queries [{:export "BarCount" :error (str "Table " categories " does not exist.")}- {:export "VenueCount" :entity map?}]- :metrics [{:id count-id :entity map?}]}- (serialize! :crowberto 200- {:queries [{:export "BarCount"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}- :aggregations [{:type "metric" :id bars-id}]}]}}- {:export "VenueCount"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}- :aggregations [{:type "metric" :id count-id}]}]}}]}))))))))--(deftest refuses-a-column-remapped-to-a-deactivated-table-test- (testing "venues.category_id displays categories.name, a join the query processor adds only when the query runs, so- the built query never names categories"- (let [categories (mt/id :categories)- venues {:type "table" :id (mt/id :venues)}]- (mt/with-column-remappings [venues.category_id categories.name]- (mt/with-temp-vals-in-db :model/Table categories {:active false}- (is (=? {:queries [{:export "VenueCategoryIds" :error (str "Table " categories " does not exist.")}- {:export "VenuePrices" :entity map?}]}- (serialize! :crowberto 200- {:queries [{:export "VenueCategoryIds"- :query {:stages [{:source venues- :fields [{:type "column" :name "CATEGORY_ID"}]}]}}- {:export "VenuePrices"- :query {:stages [{:source venues- :fields [{:type "column" :name "PRICE"}]}]}}]}))))))))--(deftest refuses-a-definition-that-builds-an-invalid-query-test- (testing "the request schema accepts what a type lets through, and lib's own checks are off in production, so the- built query is checked: an invalid one must not serialize and then fail when it runs"- (mt/with-premium-features #{:data-apps}- (mt/with-current-user (mt/user->id :crowberto)- (binding [mu.fn/*enforce* false]- (is (=? {:queries [{:export "Half" :error "The definition does not build a valid query."}- {:export "Whole" :entity map?}]}- (resource-serialization/serialize-resources- app-collection- (with-ids [{:export "Half" :query {:stages [{:source {:type :table :id (mt/id :venues)} :limit 1.5}]}}- {:export "Whole" :query {:stages [{:source {:type :table :id (mt/id :venues)} :limit 2}]}}])- []))))))))--(deftest a-public-source-serializes-as-the-private-copy-the-author-writes-test- (testing "what makes a source public or embedded is left out, since the pull refuses a copy that says it is"- (data-apps.tu/do-with-sources!- (fn [{:keys [metric-id action-id]}]- (t2/update! :model/Card :id metric-id {:public_uuid (str (random-uuid))- :made_public_by_id (mt/user->id :crowberto)- :enable_embedding true- :embedding_params {}})- (t2/update! :model/Action :id action-id {:public_uuid (str (random-uuid))- :made_public_by_id (mt/user->id :crowberto)})- (let [{:keys [actions metrics] :as response} (serialize! :crowberto 200- {:queries [{:export "VenueCount"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}- :aggregations [{:type "metric" :id metric-id}]}]}}]- :actions [action-id]})]- (is (=? {:actions [{:id action-id :entity map?}] :metrics [{:id metric-id :entity map?}]} response))- (doseq [{:keys [entity]} (concat actions metrics)]- (is (not-any? (partial contains? entity)- [:public_uuid :made_public_by_id :enable_embedding :embedding_params :embedding_type]))))))))--(deftest an-item-serialization-cannot-extract-answers-with-the-cause-test- (testing "one entity that fails inside its extraction comes back with the reason, which serialization wraps at each- level, and the rest still serialize"- (data-apps.tu/do-with-sources!- (fn [{:keys [metric-id action-id]}]- (t2/update! :model/Card :id metric-id {:visualization_settings {:broken true}})- (let [export-settings (mt/original-fn #'serdes/export-visualization-settings)]- (mt/with-dynamic-fn-redefs [serdes/export-visualization-settings (fn [settings]- (if (:broken settings)- (throw (ex-info "the metric is broken" {}))- (export-settings settings)))]- ;; called directly: the redefinition is bound on this thread, not on the one a request runs on- (is (=? {:queries [{:export "VenueCount" :entity map?}]- :actions [{:id action-id :entity map?}]- :metrics [{:id metric-id :error (str "Could not serialize Metric " metric-id ": the metric is broken")}]}- (mt/with-premium-features #{:data-apps}- (mt/with-current-user (mt/user->id :crowberto)- (resource-serialization/serialize-resources- app-collection- (with-ids [{:export "VenueCount"- :query {:stages [{:source {:type :table :id (mt/id :venues)}- :aggregations [{:type :metric :id metric-id}]}]}}])- [action-id])))))))))))--(deftest an-item-serialization-leaves-out-answers-without-a-reason-test- (testing "a card materialized by an exploration Summary is one serialization leaves out without an error"- (data-apps.tu/do-with-sources!- (fn [{:keys [metric-id]}]- (mt/with-temp [:model/Exploration {exploration-id :id} {:name "Explo" :creator_id (mt/user->id :crowberto)}- :model/Document {summary-id :id} {:name "Summary"- :creator_id (mt/user->id :crowberto)- :exploration_id exploration-id}]- (t2/update! :model/Card :id metric-id {:document_id summary-id})- (is (=? {:metrics [{:id metric-id :error (str "Could not serialize Metric " metric-id ".")}]}- (serialize! :crowberto 200- {:queries [{:export "VenueCount"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}- :aggregations [{:type "metric" :id metric-id}]}]}}]}))))))))--(deftest a-failure-of-the-serialization-is-logged-and-a-refusal-is-not-test- (testing "a refusal is the author's to act on; anything else is a failure the server keeps a trace of"- (mt/with-premium-features #{:data-apps}- (mt/with-current-user (mt/user->id :crowberto)- (mt/with-log-messages-for-level [messages [metabase-enterprise.data-apps.resource-serialization :warn]]- (is (=? {:queries [{:export "Broken" :error "No column found"}- {:export "Nothing" :error (str "Table " Integer/MAX_VALUE " does not exist.")}]}- (resource-serialization/serialize-resources- app-collection- (with-ids [{:export "Broken"- :query {:stages [{:source {:type :table :id (mt/id :venues)}- :fields [{:type :column :name "NOT_A_COLUMN"}]}]}}- {:export "Nothing"- :query {:stages [{:source {:type :table :id Integer/MAX_VALUE}}]}}])- [])))- (let [logged (filter #(str/includes? (:message %) "Could not serialize a data app resource") (messages))]- (is (= 1 (count logged)))- (is (str/includes? (:message (first logged)) "Broken"))))))))--(deftest refuses-archived-sources-test- (testing "the pull refuses an archived resource, so the serialization refuses an archived source"- (data-apps.tu/do-with-sources!- (fn [{:keys [action-id]}]- (t2/update! :model/Action :id action-id {:archived true})- (is (=? {:actions [{:id action-id :error #".*is archived.*"}]}- (serialize! :crowberto 200 {:actions [action-id]})))))))--(deftest refuses-an-action-whose-parameter-values-come-from-a-card-test- (data-apps.tu/do-with-sources!- (fn [{:keys [metric-id]}]- (let [mp (mt/metadata-provider)- action-id (actions/insert! {:name "Pick"- :type :query- :database_id (mt/id)- :dataset_query (lib/native-query mp "UPDATE venues SET name = {{name}}")- :parameters [{:id "name" :slug "name" :type :string/=- :values_source_type :card- :values_source_config {:card_id metric-id}}]})]- (is (=? {:actions [{:id action-id :error (re-pattern (str ".*reads card " metric-id ".*"))}]}- (serialize! :crowberto 200 {:actions [action-id]})))))))--(deftest serializes-a-segment-reference-by-entity-id-test- (let [mp (mt/metadata-provider)]- (mt/with-temp [:model/Segment {segment-id :id, segment-eid :entity_id}- {:name "Cheap"- :table_id (mt/id :venues)- :definition (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))- (lib/filter (lib/< (lib.metadata/field mp (mt/id :venues :price)) 3)))}]- (is (=? {:queries [{:export "Cheap"- :entity {:dataset_query {:stages [{:filters [["segment" {} segment-eid]]}]}}}]}- (serialize! :crowberto 200- {:queries [{:export "Cheap"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}- :filters [{:type "segment" :id segment-id}]}]}}]}))))))--(deftest refuses-sources-on-a-routing-destination-test- (testing "a routing destination is reachable only through its router, so the typed schema leaves out what it backs,- and so does the serialization"- (mt/with-temp [:model/Database {router-id :id} {}- :model/DatabaseRouter _ {:database_id router-id :user_attribute "region"}- :model/Database {destination-id :id} {:router_database_id router-id}]- (let [action-id (actions/insert! {:name "Rename venue"- :type :query- :database_id destination-id- :dataset_query {:lib/type :mbql/query- :database destination-id- :stages [{:lib/type :mbql.stage/native- :native "UPDATE venues SET name = 'x'"}]}})]- (is (=? {:actions [{:id action-id :error #".*is backed by a routing destination.*"}]}- (serialize! :crowberto 200 {:actions [action-id]})))))))--(deftest refuses-a-source-whose-settings-read-a-card-test- (testing "a copy keeps every card its serialization references, so a click behaviour linking a saved question is refused- like a query reading one"- (data-apps.tu/do-with-sources!- (fn [{:keys [metric-id model-id]}]- (t2/update! :model/Card :id metric-id- {:visualization_settings {:click_behavior {:type "link" :linkType "question" :targetId model-id}}})- (is (=? {:metrics [{:id metric-id :error (re-pattern (str ".*reads card " model-id ".*"))}]}- (serialize! :crowberto 200- {:queries [{:export "VenueCount"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}- :aggregations [{:type "metric" :id metric-id}]}]}}]})))))))--(deftest extracts-the-cards-and-the-actions-once-test- (testing "the cards and the actions are each extracted in one serialization query, however many the app uses"- (data-apps.tu/do-with-sources!- (fn [{:keys [metric-id action-id model-action-id]}]- (let [calls (atom [])- extract (mt/original-fn #'resource-serialization/extract-by-entity-id)]- (mt/with-dynamic-fn-redefs [resource-serialization/extract-by-entity-id (fn [model-name ids]- (swap! calls conj model-name)- (extract model-name ids))]- (is (=? {:actions [{:entity map?} {:error string?}] :metrics [{:entity map?}]}- (serialize! :crowberto 200- {:queries [{:export "VenueCount"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}- :aggregations [{:type "metric" :id metric-id}]}]}}]- :actions [action-id model-action-id]}))))- (is (= {"Card" 1 "Action" 1} (frequencies @calls))))))))--(deftest serializes-the-saved-question-an-author-writes-test- (testing "a query comes back as the saved question that holds it: named after the export, in the app's collection,- with the definition's entity ID, created by the caller, and nothing unset"- (let [{:keys [entity]} (-> (serialize! :crowberto 200- {:collection "appCollectionEntity01"- :queries [{:export "VenuesList"- :entity_id "savedQuestionEntity01"- :query {:stages [{:source {:type "table" :id (mt/id :venues)}- :limit 5}]}}]})- :queries first)]- (is (=? {:serdes/meta [{:model "Card" :id "savedQuestionEntity01" :label "venues_list"}]- :entity_id "savedQuestionEntity01"- :collection_id "appCollectionEntity01"- :name "Venues list"- :type "question"- :display "table"- :creator_id "[email protected]"- :visualization_settings {}- :parameters []- :parameter_mappings []- :dataset_query {:database (db-name)- :stages [{:source-table (table-path "VENUES") :limit 5}]}}- entity))- (is (not-any? (partial contains? entity)- [:description :collection_position :public_uuid :card_schema :archived :enable_embedding])- "what serialization leaves unset or at its default is left out, as the format omits it"))))--(deftest card-names-from-export-names-test- (are [export card-name] (= card-name (#'resource-serialization/export-name->card-name export))- "VenuesList" "Venues list"- "ordersByMonth" "Orders by month"- "top_10_products" "Top 10 products"- "Revenue" "Revenue"))--(deftest keys-come-in-the-order-serialization-writes-them-test- (testing "the printed entity comes in the order serialization writes a file, so an author keeps it"- (mt/with-premium-features #{:data-apps}- (mt/with-current-user (mt/user->id :crowberto)- (let [{:keys [entity]} (-> (resource-serialization/serialize-resources- app-collection- ;; decoded, as the endpoint hands a definition over- (with-ids [{:export "Venues" :query {:stages [{:source {:type :table :id (mt/id :venues)}}]}}])- [])- :queries first)]- (is (= [:database :stages :lib/type] (keys (:dataset_query entity))))- (is (< (.indexOf ^java.util.List (vec (keys entity)) :name)- (.indexOf ^java.util.List (vec (keys entity)) :dataset_query))))))))--(deftest omits-the-settings-an-action-leaves-unset-test- (testing "nulls inside an action's form settings, which the format omits, are left out of its serialization"- (data-apps.tu/do-with-sources!- (fn [{:keys [action-id]}]- (t2/update! :model/Action action-id- {:visualization_settings {:fields {:name {:id "name" :hidden true :description nil}}- :column_settings nil}})- (let [{:keys [entity]} (-> (serialize! :crowberto 200 {:actions [action-id]}) :actions first)]- (is (= {:fields {:name {:id "name" :hidden true}}} (:visualization_settings entity)))- (is (not (contains? entity :description))))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/schema_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/schema_test.cljindex 0a5678d4cef..836897e6441 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/schema_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/schema_test.clj@@ -5,8 +5,7 @@[metabase-enterprise.data-apps.config :as data-app.config][metabase-enterprise.data-apps.schema :as data-apps.schema][metabase.lib.core :as lib]- [metabase.util.malli.registry :as mr]- [metabase.util.yaml :as yaml]))+ [metabase.util.malli.registry :as mr]))(set! *warn-on-reflection* true)@@ -38,11 +37,6 @@(doseq [bad [0 -1 1.5 "1" "one" [1]]](is (not (valid? ::data-apps.schema/version bad)) (str "should reject: " (pr-str bad))))))-(deftest template-manifest-declares-the-supported-version-test- (testing "the scaffolding template stamps the version this Metabase serves, so a new app is never born outdated"- (is (= data-app.config/supported-app-version- (:version (yaml/parse-string (slurp "skills/metabase-data-app-setup/template/data_app.yaml")))))))-(deftest migration-upgrade-guides-cover-every-version-test(testing "the migrate skill ships one guide per upgrade up to the supported version, so an app at any older version has a path"(let [upgrades (->> (.listFiles (io/file "skills/metabase-data-app-migrate/references/upgrades"))@@ -76,7 +70,7 @@(testing "a slug is used verbatim"(is (= "inventory-2" (normalize ::data-apps.schema/slug "inventory-2"))))(testing "a slug that can't appear in a URL as-is, or collides with an API sub-route, is rejected"- (doseq [bad [nil "" "Sales" "my_app" "sales app" "-sales" "sales-" "sales\n" "repo-status" "sandbox-host"+ (doseq [bad [nil "" "Sales" "my_app" "sales app" "-sales" "sales-" "sales\n" "repo-status" "sandbox-host" "generate"(apply str (repeat 101 "a"))]](is (not (valid? ::data-apps.schema/slug bad)) (str "should reject: " (pr-str bad))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/test_util.clj b/enterprise/backend/test/metabase_enterprise/data_apps/test_util.cljindex f655774e908..f83ae1932d6 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/test_util.clj@@ -4,6 +4,7 @@[metabase-enterprise.serialization.core :as serialization][metabase-enterprise.serialization.dump :as serialization.dump][metabase.actions.core :as actions]+ [metabase.collections.core :as collections][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -186,3 +187,18 @@:model-id model-id:model-action-id (actions/insert! {:name "Create venue" :type :implicit :kind :row/create:model_id model-id})}))))++(defn do-with-library!+ "Call `f` with the IDs of the root data library (`:data-id`) and metrics library (`:metrics-id`) collections, creating the+ library for the call when the instance has none."+ [f]+ (let [existing (collections/library-collection)+ library (or existing (collections/create-library-collection!))+ children (t2/select :model/Collection :location (str "/" (:id library) "/"))+ id-of (fn [collection-type] (:id (first (filter #(= collection-type (:type %)) children))))]+ (try+ (f {:data-id (id-of collections/library-data-collection-type)+ :metrics-id (id-of collections/library-metrics-collection-type)})+ (finally+ (when-not existing+ (t2/delete! :model/Collection :id [:in (cons (:id library) (map :id children))]))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.cljindex b052e6cdb72..0358199446a 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj@@ -5,6 +5,7 @@[metabase-enterprise.remote-sync.source :as source][metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.remote-sync.test-helpers :as th]+ [metabase-enterprise.serialization.core :as serialization][metabase.models.serialization :as serdes][metabase.test :as mt][metabase.test.fixtures :as fixtures]@@ -57,9 +58,9 @@(abort-commit! [_] nil))))(version [_] "remote-tip"))))-(deftest entity->path-uses-git-separators-test+(deftest entity-file-path-uses-git-separators-test(testing "paths are git tree paths joined with / and never the host filesystem separator (#74095)"- (let [path (source/entity->path (serdes/storage-base-context) (create-test-entity "A" "a" "Card"))]+ (let [path (serialization/entity-file-path (serdes/storage-base-context) (create-test-entity "A" "a" "Card"))](is (= "collections/main/test_a.yaml" path))(is (not (str/includes? path "\\")))(testing "JGit's Windows path checker accepts the path, so pushing from a Windows host does not fail"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.cljindex 017ec92fd7a..9fd3c888a61 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj@@ -523,3 +523,21 @@{:label "tables" :key "tables"}{:label "target" :key "table-1"}]))"transforms under databases/.../schemas/ should not get _2 suffix")))))++(deftest entity-file-path-and-yaml-match-the-file-writer-test+ (testing "the file writer writes each entity at its entity-file-path, holding exactly its entity-yaml"+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]+ (mt/with-empty-h2-app-db!+ (ts/with-temp-dpc [:model/Collection parent {:name "Some Collection"}+ :model/Collection _child {:name "Child Collection" :location (format "/%d/" (:id parent))}+ :model/Card _card {:name "A Question" :collection_id (:id parent)}]+ (let [export (into [] (extract/extract {:no-settings true :no-data-model true :no-transforms true}))+ entities (filterv (comp #{"Collection" "Card"} :model last :serdes/meta) export)+ ctx (serdes/storage-base-context)]+ (storage/store! export (storage.files/file-writer dump-dir))+ (is (= 3 (count entities)))+ (doseq [entity entities+ :let [path (storage.util/entity-file-path ctx entity)]]+ (testing path+ (is (= (storage.util/entity-yaml entity)+ (slurp (io/file dump-dir path))))))))))))diff --git a/enterprise/frontend/src/embedding-sdk-package/cli/commands/data-apps.ts b/enterprise/frontend/src/embedding-sdk-package/cli/commands/data-apps.tsindex 8f92f9f8f60..f8c208514df 100644--- a/enterprise/frontend/src/embedding-sdk-package/cli/commands/data-apps.ts+++ b/enterprise/frontend/src/embedding-sdk-package/cli/commands/data-apps.ts@@ -1,7 +1,7 @@import type { Command } from "commander";import { checkResources } from "../../data-app-resources/check";-import { serializeResources } from "../../data-app-resources/serialize";+import { writeResources } from "../../data-app-resources/serialize";type AppRootOptions = { appRoot: string };@@ -19,16 +19,11 @@ export function addDataAppsCommands(program: Command) {addAppCommand(dataAppsCommand,- "print-resources",- "print what the files of the data app's collection are written from, serialized by Metabase, as JSON",- )- .argument(- "[file]",- "only the definitions in this file, relative to the app directory",- )- .action(async (file: string | undefined, { appRoot }: AppRootOptions) => {- process.stdout.write(`${await serializeResources(appRoot, file)}\n`);- });+ "write-resources",+ "regenerate the files of the data app's collection, serialized by Metabase, next to the collection's file",+ ).action(async ({ appRoot }: AppRootOptions) => {+ process.stdout.write(`${await writeResources(appRoot)}\n`);+ });addAppCommand(dataAppsCommand,diff --git a/enterprise/frontend/src/embedding-sdk-package/cli/commands/data-apps.unit.spec.ts b/enterprise/frontend/src/embedding-sdk-package/cli/commands/data-apps.unit.spec.tsindex 0d91e445659..f761a73290f 100644--- a/enterprise/frontend/src/embedding-sdk-package/cli/commands/data-apps.unit.spec.ts+++ b/enterprise/frontend/src/embedding-sdk-package/cli/commands/data-apps.unit.spec.ts@@ -14,6 +14,9 @@ import {import { addDataAppsCommands } from "./data-apps";const QUESTION = "questionEntityId00010";+const ACTION_COPY = "actionCopyEntityId001";+const QUESTION_PATH =+ "collections/data_apps/data_app/orders_questionEntityId00010.yaml";async function run(...args: string[]) {const program = new Command();@@ -42,46 +45,48 @@ describe("data app commands", () => {const printed = () =>stdout.mock.calls.map(([chunk]) => String(chunk)).join("");- it("prints the serialization of one file's definitions, relative to the app root", async () => {+ it("regenerates the collection's files for the definitions in the app root", async () => {const appRoot = appWithQuery();writeAction(appRoot,- `export const Create = defineAction({ action: { id: 51, parameters: [] } });`,+ `export const Create = defineAction({ copiedActionEntityId: "${ACTION_COPY}", action: { id: 51, parameters: [] } });`,);fs.writeFileSync(path.join(appRoot, ".env.local"),"DATA_APP_MB_URL=http://metabase.test\nDATA_APP_MB_API_KEY=mb_test_key\n",);- jest.spyOn(global, "fetch").mockResolvedValue(+ const fetchSpy = jest.spyOn(global, "fetch").mockResolvedValue(new Response(JSON.stringify({- queries: [{ export: "Orders", entity: {}, metrics: [] }],- actions: [],+ queries: [+ {+ file: "orders_questionEntityId00010.yaml",+ yaml: "name: Orders\n",+ },+ ],+ actions: [{ file: "create.yaml", yaml: "name: Create\n" }],metrics: [],}),),);- await run(- "print-resources",- "queries/orders.query.ts",- "--app-root",- appRoot,- );+ await run("write-resources", "--app-root", appRoot);- expect(JSON.parse(printed())).toEqual({- queries: [+ expect(JSON.parse(String(fetchSpy.mock.calls[0][1]?.body)).actions).toEqual(+ [{- export: "Orders",- file: "queries/orders.query.ts",- savedQuestionEntityId: QUESTION,- entity: {},- metrics: [],+ action_id: 51,+ entity_id: ACTION_COPY,+ collection_id: "appCollectionEntity01",},],- actions: [],- metrics: [],- });+ );+ expect(fs.readFileSync(path.join(appRoot, QUESTION_PATH), "utf8")).toBe(+ "name: Orders\n",+ );+ expect(printed()).toBe(+ `Wrote ${QUESTION_PATH}\nWrote collections/data_apps/data_app/create.yaml\n`,+ );});it("confirms resources that back every definition", async () => {diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-resources/README.md b/enterprise/frontend/src/embedding-sdk-package/data-app-resources/README.mdindex 570c9a43e21..95872690302 100644--- a/enterprise/frontend/src/embedding-sdk-package/data-app-resources/README.md+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-resources/README.md@@ -7,13 +7,12 @@ fail on permissions.The files of the app's collection make that true. Under the repository's `collections/data_apps/`, thefolder of the `data-apps` collection namespace, they hold serdes YAML for the app's collection, a saved-question per query, and copies of the actions the app runs and of the metrics its queries use. The-author (in practice, an agent following the data-app skills) writes that YAML from the app's-definitions. Nothing changes in Metabase until the repository is pulled: the pull loads the files as+question per query, and copies of the actions the app runs and of the metrics its queries use. Metabase+serializes everything but the collection's own file from the app's definitions, and the CLI writes it. Nothing changes in Metabase until the repository is pulled: the pull loads the files asserialized content, like everything else the repository holds, so a local experiment can't break therunning app, and the saved questions never get ahead of the app code that is deployed.-`check-resources` reads nothing from Metabase; `print-resources` asks it for the serialization.+`check-resources` reads nothing from Metabase; `write-resources` asks it for the files.## What an app declares@@ -44,25 +43,28 @@ app that isn't under `data_apps/` is its own root, so its files sit under its owA data app runs only query actions that belong to no model, which the typed schema lists under`schema.actions`. Such an action's permissions resolve through its own collection, so making itreachable means copying it into the app's collection. Metrics a query uses are copied for the same-reason, and the question reads the copy. Every saved question and copy is written from what-`print-resources` prints, whose references are already in the form the YAML uses, with new entity IDs-and references rewritten to the copies.--## `print-resources`--`embedding-sdk-react data-apps print-resources [file]` (`serialize.ts`) sends the evaluated definitions (all-of them, or those in `file`, relative to the app directory) to `POST /api/apps/serialize-resources`, with the-instance and API key from `.env.local`, and prints the answer as JSON: the saved question Metabase-writes for each `defineQuery` definition, each `defineAction`'s source action, and the metrics the-queries aggregate, all as serialization writes them, each beside the definition's file and entity ID.-An action that belongs to a model comes back with an error. The endpoint answers only a superuser, so-the API key must be one in the Administrators group. The saved question is complete: named after the export, in the-collection `data_app.yaml` names, with the definition's `savedQuestionEntityId`, created by the API-key's user, and holding the query Metabase builds with the same `createTestQuery` code the dev preview-runs. Every entity comes in the key order serialization writes a file and without the keys serialization-leaves unset, which the format omits, so the author transcribes rather than composes. An item that can't-be built or copied comes back with its `error`; the rest still come back. The command refuses to run-before `data_app.yaml` names the app's collection, since the saved questions are written into it.+reason, and the question reads the copy. Every saved question and copy is written by+`write-resources`, with the entity IDs the definitions carry and the metric references already+rewritten to the copies.++## `write-resources`++`embedding-sdk-react data-apps write-resources` (`serialize.ts`) sends the evaluated definitions to+`POST /api/apps/generate/resources`, with the instance and API key from `.env.local`, and regenerates the app's+collection folder from the files it answers with: the saved question Metabase builds for each+`defineQuery` definition, the copy of each `defineAction`'s action, and the copies of the metrics the+queries aggregate. Each answer is a file name and the YAML text exactly as a remote-sync export writes it;+the command writes it into the folder beside the collection's file (`<collection>/`). It first deletes every+card and action file whose `collection_id` is the app's collection, wherever it sits under+`collections/data_apps/`, so a copy no definition needs any more disappears, and a file name that would+leave the folder is refused. It prints a `Wrote <path>` line per file. The collection's own file must exist+already; the instance needs neither the collection nor a pull before the command runs. The endpoint+answers only a superuser, so the API key must be one in the Administrators group. The saved question is+named after the export, in the app's collection, with the definition's `savedQuestionEntityId`, and holds+the query Metabase builds with the same `createTestQuery` code the dev preview runs. An action that belongs+to a model, or an item that can't be built or copied, comes back with an `error`: then the command writes+and deletes nothing and fails listing every error. It refuses to run before `data_app.yaml` names the+app's collection, and before every definition carries its entity ID.## `check-resources`@@ -89,15 +91,16 @@ Nothing checks that a saved question's query matches its definition.evaluates it, and takes every exported object as a definition: `defineQuery` and `defineAction`return their argument as is, and the two directories hold nothing else. A definition a second filere-exports keeps its identity through the single bundle, so it counts once, for the first file that-exports it; `print-resources <file>` reads that file's exports alone, so a barrel doesn't claim them.+exports it.Anything that isn't a definition is rejected by the serialization endpoint's schema, not here.Discovery refuses what makes a definition unusable on its own or against the others: an action thatdoesn't reference a generated action, two definitions of one source action, two claiming one entity-ID. Every command needs that, `print-resources` included. What a definition lacks against the app's-collection files is `check-resources`' to report. A definition's own entity ID is the exception: the saved-question is printed with it, so `print-resources` refuses a `defineQuery` without `savedQuestionEntityId`-and names it, and the author generates the ID with `npx representations generate-entity-id` first.+ID. Every command needs that, `write-resources` included. What a definition lacks against the app's+collection files is `check-resources`' to report. A definition's own entity ID is the exception: its file is+written with it, so `write-resources` refuses a `defineQuery` without `savedQuestionEntityId` or a+`defineAction` without `copiedActionEntityId` and names it, and the author generates the ID with+`npx representations generate-entity-id` first.## Dev preview vs production builddiff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-resources/discover.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-resources/discover.tsindex cccabfc1485..8f0e506b459 100644--- a/enterprise/frontend/src/embedding-sdk-package/data-app-resources/discover.ts+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-resources/discover.ts@@ -111,7 +111,7 @@ async function evaluateFiles(directory: string, filePaths: string[]) {}));}-/** Narrows discovery to one definition file, the one `print-resources <file>` is asked about. */+/** Narrows discovery to one definition file. */interface DiscoveryOptions {filePath?: string;}diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-resources/serialize.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-resources/serialize.tsindex 0f1d7c961b5..dd98cee82f2 100644--- a/enterprise/frontend/src/embedding-sdk-package/data-app-resources/serialize.ts+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-resources/serialize.ts@@ -1,3 +1,4 @@+import fs from "node:fs";import path from "node:path";import { readManifest } from "../data-app-dev/config/read-manifest";@@ -10,38 +11,49 @@ import {} from "./discover";import { getMetabaseCredentials } from "./env";import { isObject } from "./guards";-import { COLLECTIONS_DIR } from "./resources";-import type { DiscoveredQuery } from "./types";+import { COLLECTIONS_DIR, readResources, repoRootOf } from "./resources";+import type { DiscoveredAction, DiscoveredQuery } from "./types";-type IdentifiedQuery = DiscoveredQuery & { savedQuestionEntityId: string };--const isIdentified = (query: DiscoveredQuery): query is IdentifiedQuery =>- query.savedQuestionEntityId !== undefined;+type SerializedFile = { file: string; yaml: string } | { error: string };interface SerializedResources {- queries: Record<string, unknown>[];- actions: Record<string, unknown>[];- metrics: unknown[];+ queries: SerializedFile[];+ actions: SerializedFile[];+ metrics: SerializedFile[];}-const isObjectArray = (value: unknown): value is Record<string, unknown>[] =>- Array.isArray(value) && value.every(isObject);+const isSerializedFile = (value: unknown): value is SerializedFile =>+ isObject(value) &&+ ((typeof value.file === "string" && typeof value.yaml === "string") ||+ typeof value.error === "string");++const isSerializedFiles = (value: unknown): value is SerializedFile[] =>+ Array.isArray(value) && value.every(isSerializedFile);function isSerializedResources(value: unknown): value is SerializedResources {return (isObject(value) &&- isObjectArray(value.queries) &&- isObjectArray(value.actions) &&- Array.isArray(value.metrics)+ isSerializedFiles(value.queries) &&+ isSerializedFiles(value.actions) &&+ isSerializedFiles(value.metrics));}+export function exportNameToCardName(exportName: string) {+ const words = exportName+ .replace(/([a-z0-9])([A-Z])/g, "$1 $2")+ .replace(/[\s_]+/g, " ")+ .toLowerCase();++ return words.charAt(0).toUpperCase() + words.slice(1);+}+async function requestSerialization(appRoot: string,- body: { collection: string; queries: unknown[]; actions: number[] },+ body: { queries: unknown[]; actions: unknown[] },): Promise<SerializedResources> {const { metabaseUrl, apiKey } = getMetabaseCredentials(appRoot);- const response = await fetch(`${metabaseUrl}/api/apps/serialize-resources`, {+ const response = await fetch(`${metabaseUrl}/api/apps/generate/resources`, {method: "POST",headers: { "Content-Type": "application/json", "X-API-Key": apiKey },body: JSON.stringify(body),@@ -62,101 +74,139 @@ async function requestSerialization(return serialized;}-/**- * What the files of the app's collection are written from, as JSON, for the- * definitions in `file` (relative to the app root, as files are printed) or- * every definition: the saved question Metabase writes for each `defineQuery`- * definition, in the collection `data_app.yaml` names and with the- * definition's entity ID, each `defineAction`'s action, and the metrics the- * queries aggregate, all as serialization writes them.- * One request to the Metabase instance and API key in `.env.local`.- */-export async function serializeResources(appDirectory: string, file?: string) {- const appRoot = path.resolve(appDirectory);- const filePath = file === undefined ? undefined : path.resolve(appRoot, file);- const queries = await discoverQueries(appRoot, { filePath });- const actions = await discoverActions(appRoot, { filePath });+function missingEntityIds(+ appRoot: string,+ queries: DiscoveredQuery[],+ actions: DiscoveredAction[],+) {+ return [+ ...queries+ .filter((query) => query.savedQuestionEntityId === undefined)+ .map(+ (query) =>+ `${getRelativeDefinitionLocation(appRoot, query)} has no savedQuestionEntityId. Generate one with \`npx representations generate-entity-id\` and set it first: its saved question is written with it.`,+ ),+ ...actions+ .filter((action) => action.copiedActionEntityId === undefined)+ .map(+ (action) =>+ `${getRelativeDefinitionLocation(appRoot, action)} has no copiedActionEntityId. Generate one with \`npx representations generate-entity-id\` and set it first: its copy is written with it.`,+ ),+ ];+}- if (filePath !== undefined && queries.length + actions.length === 0) {+function collectionDirectory(appRoot: string, collection: string) {+ const { repoRoot, collectionPath, files } = readResources(+ appRoot,+ collection,+ );++ if (collectionPath === undefined) {throw new Error(- `${path.relative(appRoot, filePath)} has no defineQuery or defineAction definitions.`,+ `No file under ${COLLECTIONS_DIR}/ holds the collection ${collection} that data_app.yaml names. Write it first: the files are written next to it.`,);}- const collection = readManifest(appRoot)?.manifest.collection;+ return {+ directory: path.join(repoRoot, collectionPath.replace(/\.yaml$/, "")),+ existing: files.map((file) => path.join(repoRoot, file.path)),+ };+}- if (collection === undefined) {+function targetFile(directory: string, file: { file: string; yaml: string }) {+ const target = path.join(directory, file.file);++ if (path.dirname(target) !== directory) {throw new Error(- `No data_app.yaml in ${appRoot} names the app's collection. Write the collection under ${COLLECTIONS_DIR}/ and name its entity ID as \`collection\` first: the saved questions are written into it.`,+ `The serialization holds a file outside the collection: ${file.file}`,);}- // The saved question is printed with the definition's entity ID, so the file is complete as printed.- const unidentified = queries.filter((query) => !isIdentified(query));+ return { target, yaml: file.yaml };+}++/**+ * Regenerates the files of the app's collection: the saved question Metabase+ * builds for each `defineQuery` definition, the copy of each `defineAction`'s+ * action, and the copies of the metrics the queries aggregate, written next to+ * the collection's file in place of the cards and actions it held. Writes+ * nothing when Metabase couldn't serialize an item, and throws listing why.+ * One request to the Metabase instance and API key in `.env.local`.+ */+export async function writeResources(appDirectory: string) {+ const appRoot = path.resolve(appDirectory);+ const queries = await discoverQueries(appRoot);+ const actions = await discoverActions(appRoot);+ const collection = readManifest(appRoot)?.manifest.collection;- if (unidentified.length > 0) {+ if (collection === undefined) {throw new Error(- unidentified- .map(- (query) =>- `${getRelativeDefinitionLocation(appRoot, query)} has no savedQuestionEntityId. Generate one with \`npx representations generate-entity-id\` and set it first: its saved question is printed with it.`,- )- .join("\n"),+ `No data_app.yaml in ${appRoot} names the app's collection. Write the collection under ${COLLECTIONS_DIR}/ and name its entity ID as \`collection\` first: the files are written into it.`,);}- const identified = queries.filter(isIdentified);+ const { directory, existing } = collectionDirectory(appRoot, collection);+ const missing = missingEntityIds(appRoot, queries, actions);++ if (missing.length > 0) {+ throw new Error(missing.join("\n"));+ }const serialized = await requestSerialization(appRoot, {- collection,- queries: identified.map(({ exportName, query, savedQuestionEntityId }) => {+ queries: queries.map(({ exportName, query, savedQuestionEntityId }) => {const { [QUERY_DEFINITIONS.idKey]: _entityId, ...definition } = query;return {- export: exportName,- entity_id: savedQuestionEntityId,+ name: exportNameToCardName(exportName),query: { stages: [definition] },+ entity_id: savedQuestionEntityId,+ collection_id: collection,};}),- actions: actions.map(({ sourceActionId }) => sourceActionId),+ actions: actions.map(({ sourceActionId, copiedActionEntityId }) => ({+ action_id: sourceActionId,+ entity_id: copiedActionEntityId,+ collection_id: collection,+ })),});- const serializedActions = new Map(- serialized.actions.map((action) => [action.id, action]),- );-- if (serialized.queries.length !== identified.length) {+ if (+ serialized.queries.length !== queries.length ||+ serialized.actions.length !== actions.length+ ) {throw new Error(- `The serialization response holds ${serialized.queries.length} queries; ${identified.length} were requested.`,+ "The serialization response doesn't answer every definition.",);}- const missingActions = actions- .map(({ sourceActionId }) => sourceActionId)- .filter((id) => !serializedActions.has(id));+ const labelled = [+ ...serialized.queries.map((result, index) => ({+ label: getRelativeDefinitionLocation(appRoot, queries[index]),+ result,+ })),+ ...serialized.actions.map((result, index) => ({+ label: getRelativeDefinitionLocation(appRoot, actions[index]),+ result,+ })),+ ...serialized.metrics.map((result) => ({ label: "A metric", result })),+ ];++ const errors = labelled.flatMap(({ label, result }) =>+ "error" in result ? [`${label}: ${result.error}`] : [],+ );- if (missingActions.length > 0) {- throw new Error(- `The serialization response is missing action ${missingActions.join(", ")}.`,- );+ if (errors.length > 0) {+ throw new Error(errors.join("\n"));}- return JSON.stringify(- {- queries: identified.map((query, index) => ({- export: query.exportName,- file: path.relative(appRoot, query.filePath),- savedQuestionEntityId: query.savedQuestionEntityId,- ...serialized.queries[index],- })),- actions: actions.map((action) => ({- export: action.exportName,- file: path.relative(appRoot, action.filePath),- copiedActionEntityId: action.copiedActionEntityId ?? null,- ...serializedActions.get(action.sourceActionId),- })),- metrics: serialized.metrics,- },- null,- 2,+ const targets = labelled.flatMap(({ result }) =>+ "error" in result ? [] : [targetFile(directory, result)],);++ existing.forEach((filePath) => fs.rmSync(filePath, { force: true }));+ fs.mkdirSync(directory, { recursive: true });+ targets.forEach(({ target, yaml }) => fs.writeFileSync(target, yaml));++ return targets+ .map(({ target }) => `Wrote ${path.relative(repoRootOf(appRoot), target)}`)+ .join("\n");}diff --git a/enterprise/frontend/src/embedding-sdk-package/data-app-resources/tests/serialize.unit.spec.ts b/enterprise/frontend/src/embedding-sdk-package/data-app-resources/tests/serialize.unit.spec.tsindex 0ced06d55de..0e859140d74 100644--- a/enterprise/frontend/src/embedding-sdk-package/data-app-resources/tests/serialize.unit.spec.ts+++ b/enterprise/frontend/src/embedding-sdk-package/data-app-resources/tests/serialize.unit.spec.ts@@ -1,30 +1,31 @@import fs from "node:fs";import path from "node:path";-import { serializeResources } from "../serialize";+import { exportNameToCardName, writeResources } from "../serialize";-import { makeApp, setupResourceTests, writeAction, writeQuery } from "./setup";+import {+ COLLECTION,+ makeApp,+ setupResourceTests,+ writeAction,+ writeQuery,+ writeResource,+} from "./setup";const QUESTION = "questionEntityId00010";const ACTION_COPY = "actionCopyEntityId001";-const COLLECTION = "appCollectionEntity01";+const COLLECTION_DIR = "collections/data_apps/data_app";+const QUESTION_PATH = `${COLLECTION_DIR}/orders_${QUESTION}.yaml`;+const ACTION_PATH = `${COLLECTION_DIR}/create_${ACTION_COPY}.yaml`;+const METRIC_PATH = `${COLLECTION_DIR}/revenue_metricCopyEntityId0001.yaml`;const SERIALIZED = {- queries: [- {- export: "Orders",- entity: {- entity_id: QUESTION,- collection_id: COLLECTION,- name: "Orders",- dataset_query: { database: "Sample Database" },- },- metrics: [],- },+ queries: [{ file: `orders_${QUESTION}.yaml`, yaml: "name: Orders\n" }],+ actions: [{ file: `create_${ACTION_COPY}.yaml`, yaml: "name: Create\n" }],+ metrics: [+ { file: "revenue_metricCopyEntityId0001.yaml", yaml: "name: Revenue\n" },],- actions: [{ id: 51, entity: { entity_id: "sourceActionEntity051" } }],- metrics: [],};function appWithDefinitions() {@@ -55,7 +56,7 @@ function mockSerialization(response: Response) {describe("serializing what resources are written from", () => {setupResourceTests();- it("refuses to print before the manifest names the app's collection", async () => {+ it("refuses to write before the manifest names the app's collection", async () => {const appRoot = appWithDefinitions();fs.writeFileSync(path.join(appRoot, "data_app.yaml"),@@ -65,13 +66,13 @@ describe("serializing what resources are written from", () => {new Response(JSON.stringify(SERIALIZED)),);- await expect(serializeResources(appRoot)).rejects.toThrow(+ await expect(writeResources(appRoot)).rejects.toThrow("names the app's collection",);expect(fetchSpy).not.toHaveBeenCalled();});- it("refuses to print a query that has no saved question ID yet", async () => {+ it("refuses to write a query that has no saved question ID yet", async () => {const appRoot = appWithDefinitions();fs.appendFileSync(path.join(appRoot, "queries/orders.query.ts"),@@ -81,82 +82,176 @@ describe("serializing what resources are written from", () => {new Response(JSON.stringify(SERIALIZED)),);- await expect(serializeResources(appRoot)).rejects.toThrow(+ await expect(writeResources(appRoot)).rejects.toThrow("queries/orders.query.ts:Products has no savedQuestionEntityId.",);expect(fetchSpy).not.toHaveBeenCalled();});- it("sends the definitions with their IDs, the app's collection, and the action IDs in one request, and prints the serialization beside each definition", async () => {+ it("refuses to write an action that has no copied action ID yet", async () => {+ const appRoot = appWithDefinitions();+ writeAction(+ appRoot,+ `export const Create = defineAction({ action: { id: 51, parameters: [] } });`,+ );+ const fetchSpy = mockSerialization(+ new Response(JSON.stringify(SERIALIZED)),+ );++ await expect(writeResources(appRoot)).rejects.toThrow(+ "actions/orders.action.ts:Create has no copiedActionEntityId.",+ );+ expect(fetchSpy).not.toHaveBeenCalled();+ });++ it("sends the definitions with their IDs and the app's collection in one request, and writes every returned file at its path", async () => {const appRoot = appWithDefinitions();const fetchSpy = mockSerialization(new Response(JSON.stringify(SERIALIZED)),);- const printed = JSON.parse(await serializeResources(appRoot));+ const output = await writeResources(appRoot);expect(fetchSpy).toHaveBeenCalledTimes(1);const [url, init] = fetchSpy.mock.calls[0];- expect(url).toBe("http://metabase.test/api/apps/serialize-resources");+ expect(url).toBe("http://metabase.test/api/apps/generate/resources");expect(init?.headers).toEqual({"Content-Type": "application/json","X-API-Key": "mb_test_key",});expect(JSON.parse(String(init?.body))).toEqual({- collection: COLLECTION,queries: [{- export: "Orders",- entity_id: QUESTION,+ name: "Orders",query: { stages: [{ source: { type: "table", id: 1 }, limit: 5 }] },- },- ],- actions: [51],- });- expect(printed).toEqual({- queries: [- {- export: "Orders",- file: "queries/orders.query.ts",- savedQuestionEntityId: QUESTION,- entity: SERIALIZED.queries[0].entity,- metrics: [],+ entity_id: QUESTION,+ collection_id: COLLECTION,},],actions: [- {- export: "Create",- file: "actions/orders.action.ts",- copiedActionEntityId: ACTION_COPY,- id: 51,- entity: { entity_id: "sourceActionEntity051" },- },+ { action_id: 51, entity_id: ACTION_COPY, collection_id: COLLECTION },],- metrics: [],});+ expect(output).toBe(+ [QUESTION_PATH, ACTION_PATH, METRIC_PATH]+ .map((written) => `Wrote ${written}`)+ .join("\n"),+ );+ expect(fs.readFileSync(path.join(appRoot, QUESTION_PATH), "utf8")).toBe(+ "name: Orders\n",+ );+ expect(fs.readFileSync(path.join(appRoot, ACTION_PATH), "utf8")).toBe(+ "name: Create\n",+ );+ expect(fs.readFileSync(path.join(appRoot, METRIC_PATH), "utf8")).toBe(+ "name: Revenue\n",+ );+ });++ it("writes into the repository above an app under data_apps/", async () => {+ const appRoot = makeApp({ underDataApps: true });+ fs.writeFileSync(+ path.join(appRoot, ".env.local"),+ "DATA_APP_MB_URL=http://metabase.test/\nDATA_APP_MB_API_KEY=mb_test_key\n",+ );+ writeQuery(+ appRoot,+ `export const Orders = defineQuery({ savedQuestionEntityId: "${QUESTION}", source: { type: "table", id: 1 } });`,+ );+ mockSerialization(+ new Response(JSON.stringify({ ...SERIALIZED, actions: [], metrics: [] })),+ );++ await writeResources(appRoot);++ expect(fs.existsSync(path.join(appRoot, "..", "..", QUESTION_PATH))).toBe(+ true,+ );+ });++ it("replaces the cards and actions the collection held", async () => {+ const appRoot = appWithDefinitions();+ const stale = writeResource(appRoot, "other_place/stale.yaml", {+ name: "Stale",+ type: "question",+ entity_id: "staleEntityId0000001",+ "serdes/meta": [{ model: "Card", id: "staleEntityId0000001" }],+ });+ mockSerialization(new Response(JSON.stringify(SERIALIZED)));++ await writeResources(appRoot);++ expect(fs.existsSync(stale)).toBe(false);+ expect(+ fs.existsSync(path.join(appRoot, "collections/data_apps/data_app.yaml")),+ ).toBe(true);});- it("sends only the definitions in the given file", async () => {+ it("writes nothing and throws every per-item error", async () => {const appRoot = appWithDefinitions();- const fetchSpy = mockSerialization(- new Response(JSON.stringify({ ...SERIALIZED, actions: [] })),+ const stale = writeResource(appRoot, "stale.yaml", {+ name: "Stale",+ type: "question",+ entity_id: "staleEntityId0000001",+ "serdes/meta": [{ model: "Card", id: "staleEntityId0000001" }],+ });+ mockSerialization(+ new Response(+ JSON.stringify({+ queries: [{ error: "Unknown database" }],+ actions: [{ file: `create_${ACTION_COPY}.yaml`, yaml: "x: 1\n" }],+ metrics: [{ error: "Unknown metric" }],+ }),+ ),);- await serializeResources(appRoot, "queries/orders.query.ts");+ await expect(writeResources(appRoot)).rejects.toThrow(+ [+ "queries/orders.query.ts:Orders: Unknown database",+ "A metric: Unknown metric",+ ].join("\n"),+ );+ expect(fs.existsSync(stale)).toBe(true);+ expect(fs.existsSync(path.join(appRoot, ACTION_PATH))).toBe(false);+ });- expect(JSON.parse(String(fetchSpy.mock.calls[0][1]?.body))).toEqual({- collection: COLLECTION,- queries: [expect.objectContaining({ export: "Orders" })],- actions: [],+ it("refuses a returned file that would leave the collection's folder", async () => {+ const appRoot = appWithDefinitions();+ const stale = writeResource(appRoot, "stale.yaml", {+ name: "Stale",+ type: "question",+ entity_id: "staleEntityId0000001",+ "serdes/meta": [{ model: "Card", id: "staleEntityId0000001" }],});+ mockSerialization(+ new Response(+ JSON.stringify({+ ...SERIALIZED,+ queries: [{ file: "../escaped.yaml", yaml: "name: Orders\n" }],+ }),+ ),+ );++ await expect(writeResources(appRoot)).rejects.toThrow(+ "The serialization holds a file outside the collection: ../escaped.yaml",+ );+ expect(+ fs.existsSync(path.join(appRoot, "collections/data_apps/escaped.yaml")),+ ).toBe(false);+ expect(fs.existsSync(stale)).toBe(true);});- it("fails for a file with no definitions", async () => {+ it("refuses to write before the collection has a file", async () => {const appRoot = appWithDefinitions();+ fs.rmSync(path.join(appRoot, "collections"), { recursive: true });+ const fetchSpy = mockSerialization(+ new Response(JSON.stringify(SERIALIZED)),+ );- await expect(serializeResources(appRoot, "src/App.tsx")).rejects.toThrow(- "src/App.tsx has no defineQuery or defineAction definitions.",+ await expect(writeResources(appRoot)).rejects.toThrow(+ `No file under collections/data_apps/ holds the collection ${COLLECTION}`,);+ expect(fetchSpy).not.toHaveBeenCalled();});it("fails without the Metabase instance and API key", async () => {@@ -171,7 +266,7 @@ describe("serializing what resources are written from", () => {delete process.env.DATA_APP_MB_API_KEY;try {- await expect(serializeResources(appRoot)).rejects.toThrow(+ await expect(writeResources(appRoot)).rejects.toThrow("DATA_APP_MB_URL and DATA_APP_MB_API_KEY must be set, in the repo-root .env.local or the environment.",);} finally {@@ -184,25 +279,14 @@ describe("serializing what resources are written from", () => {}});- it("fails when the response has a query too few", async () => {+ it("fails when the response doesn't answer every definition", async () => {const appRoot = appWithDefinitions();mockSerialization(new Response(JSON.stringify({ ...SERIALIZED, queries: [] })),);- await expect(serializeResources(appRoot)).rejects.toThrow(- "The serialization response holds 0 queries; 1 were requested.",- );- });-- it("fails when the response lacks a requested action", async () => {- const appRoot = appWithDefinitions();- mockSerialization(- new Response(JSON.stringify({ ...SERIALIZED, actions: [] })),- );-- await expect(serializeResources(appRoot)).rejects.toThrow(- "The serialization response is missing action 51.",+ await expect(writeResources(appRoot)).rejects.toThrow(+ "The serialization response doesn't answer every definition.",);});@@ -210,8 +294,13 @@ describe("serializing what resources are written from", () => {const appRoot = appWithDefinitions();mockSerialization(new Response("Unauthenticated", { status: 401 }));- await expect(serializeResources(appRoot)).rejects.toThrow(+ await expect(writeResources(appRoot)).rejects.toThrow("The serialization request failed (401): Unauthenticated",);});++ it("turns an export name into a card name", () => {+ expect(exportNameToCardName("OrdersByMonth")).toBe("Orders by month");+ expect(exportNameToCardName("monthly_revenue")).toBe("Monthly revenue");+ });});diff --git a/enterprise/frontend/src/embedding-sdk-package/hooks/public/use-metabase-query/tests/dynamic-query.unit.spec.tsx b/enterprise/frontend/src/embedding-sdk-package/hooks/public/use-metabase-query/tests/dynamic-query.unit.spec.tsxindex 18f2df78109..0999444fe54 100644--- a/enterprise/frontend/src/embedding-sdk-package/hooks/public/use-metabase-query/tests/dynamic-query.unit.spec.tsx+++ b/enterprise/frontend/src/embedding-sdk-package/hooks/public/use-metabase-query/tests/dynamic-query.unit.spec.tsx@@ -43,7 +43,7 @@ describe("a table source without a saved question", () => {QUERY_WITHOUT_SAVED_QUESTION,),).rejects.toThrow(- "This query has no saved question. Write it to the app's collection under `collections/data_apps/`",+ "This query has no saved question. Set its `savedQuestionEntityId` in `defineQuery(...)` in `queries/`",);});diff --git a/frontend/src/embedding-sdk-bundle/lib/create-metabase-query/index.ts b/frontend/src/embedding-sdk-bundle/lib/create-metabase-query/index.tsindex a7d315d9b11..3db80d15f0a 100644--- a/frontend/src/embedding-sdk-bundle/lib/create-metabase-query/index.ts+++ b/frontend/src/embedding-sdk-bundle/lib/create-metabase-query/index.ts@@ -79,7 +79,7 @@ async function toSourceInput(if (entityId === undefined) {// An app's viewers can only read the published cards, so a table source 403s.throw new Error(- "This query has no saved question. Write it to the app's collection under `collections/data_apps/`, set its `savedQuestionEntityId` in `defineQuery(...)` in `queries/`, run `npm run check-resources`, commit, and rebuild.",+ "This query has no saved question. Set its `savedQuestionEntityId` in `defineQuery(...)` in `queries/`, run `npm run write-resources` and `npm run check-resources`, commit, and rebuild.",);}diff --git a/frontend/src/embedding-sdk-bundle/lib/execute-action.ts b/frontend/src/embedding-sdk-bundle/lib/execute-action.tsindex 475dd6a22ce..64251ae1ee3 100644--- a/frontend/src/embedding-sdk-bundle/lib/execute-action.ts+++ b/frontend/src/embedding-sdk-bundle/lib/execute-action.ts@@ -72,7 +72,7 @@ function toExecutableActionId(input: SdkActionInput): SdkActionId {if (input.copiedActionEntityId === undefined) {throw new Error(- "This action has no copy. Copy it into the app's collection under `collections/data_apps/`, set its `copiedActionEntityId`, run `npm run check-resources`, commit, and rebuild.",+ "This action has no copy. Set its `copiedActionEntityId` in `defineAction(...)` in `actions/`, run `npm run write-resources` and `npm run check-resources`, commit, and rebuild.",);}diff --git a/frontend/src/embedding-sdk-bundle/lib/execute-action.unit.spec.ts b/frontend/src/embedding-sdk-bundle/lib/execute-action.unit.spec.tsindex bc366104fc6..08ff6c7f25a 100644--- a/frontend/src/embedding-sdk-bundle/lib/execute-action.unit.spec.ts+++ b/frontend/src/embedding-sdk-bundle/lib/execute-action.unit.spec.ts@@ -81,7 +81,7 @@ describe("executeAction", () => {await expect(executeAction(setup())({ actionId: { action: { id: AUTHORED_ID } } }),).rejects.toThrow(- "This action has no copy. Copy it into the app's collection under `collections/data_apps/`",+ "This action has no copy. Set its `copiedActionEntityId` in `defineAction(...)` in `actions/`",);});diff --git a/skills/metabase-data-app-actions/SKILL.md b/skills/metabase-data-app-actions/SKILL.mdindex dccf42829a4..989a9793736 100644--- a/skills/metabase-data-app-actions/SKILL.md+++ b/skills/metabase-data-app-actions/SKILL.md@@ -17,8 +17,6 @@ A Metabase **action** is a saved, parameterized SQL write against the data wareh## What's in the schema (and what isn't)-Action entries are only generated when the typed schema includes actions. Before writing action-invoking code, make sure the schema was generated with `include-actions=true`; with `database=<name-or-id>&include-actions=true`, Metabase includes the actions for that database only.-Before writing any action-invoking code, enumerate what's available under `schema.actions`. The schema is your **complete** catalog of the actions the app can run. Anything not present doesn't exist as far as the Data App is concerned: when the app needs a write the schema lacks, ask the user to create it as a query action without a model (`POST /api/action` with `type: "query"` and no `model_id`; actions created in the Metabase UI belong to a model and are never listed), on a database with actions enabled, then regenerate the schema.## The hookdiff --git a/skills/metabase-data-app-migrate/SKILL.md b/skills/metabase-data-app-migrate/SKILL.mdindex 8c30ee0cef6..f117a55747f 100644--- a/skills/metabase-data-app-migrate/SKILL.md+++ b/skills/metabase-data-app-migrate/SKILL.md@@ -64,12 +64,7 @@ grep -E '^version:' data_app.yaml || echo "version: 1"ls <skill-dir>/references/upgrades/ | sed -nE 's/^v[0-9]+-to-v([0-9]+)\.md$/\1/p' | sort -n | tail -1 | grep . || echo 1```-`<skill-dir>` is the directory this SKILL.md was loaded from. The target must-equal the `version:` in the data-app scaffolding template installed alongside-this skill (`<skills-dir>/*/template/data_app.yaml`), when one is present; if-they differ, the skills come from different Metabase releases. **Stop** and tell-the user to reinstall all data-app skills with the command shown under-Admin > Data apps. Wait for the answer.+`<skill-dir>` is the directory this SKILL.md was loaded from.Then decide:diff --git a/skills/metabase-data-app-migrate/references/maintainer-checklist.md b/skills/metabase-data-app-migrate/references/maintainer-checklist.mdindex 225142e1d75..9ad81fe3572 100644--- a/skills/metabase-data-app-migrate/references/maintainer-checklist.md+++ b/skills/metabase-data-app-migrate/references/maintainer-checklist.md@@ -20,8 +20,7 @@ deletes the alias bumps the contract version and ships the upgrade.1. `enterprise/backend/src/metabase_enterprise/data_apps/config.clj`:`supported-app-version` to `N+1`. `initial-app-version` stays 1.-2. `skills/metabase-data-app-setup/template/data_app.yaml`: `version: N+1`.- Update the template's source if the contract change touches it.+2. Update `skills/metabase-data-app-setup/template/` if the contract change touches it.3. Every e2e fixture manifest that declares a version(`e2e/support/assets/data-apps/*/data_app.yaml`,`e2e/support/assets/example_synced_data_apps/data_apps/*/data_app.yaml`,@@ -38,6 +37,6 @@ deletes the alias bumps the contract version and ships the upgrade.`sandbox.ts` fallback point; that removal is the breaking change.7. `enterprise/frontend/src/embedding-sdk-package/CHANGELOG.md`: a "Data appscontract v<N+1>" entry linking the upgrade guide.-8. Run `./bin/test-agent :only '[metabase-enterprise.data-apps.config-test]'`.- Two tests fail until steps 1, 2, and 4 agree: the template must declare the- supported version, and the upgrade guides must cover every version from 1 up to it.+8. Run `./bin/test-agent :only '[metabase-enterprise.data-apps.schema-test]'`.+ It fails until steps 1 and 4 agree: the upgrade guides must cover every+ version from 1 up to the supported one.diff --git a/skills/metabase-data-app-semantic-layer/SKILL.md b/skills/metabase-data-app-semantic-layer/SKILL.mdindex 767acbd0357..e899fc9e119 100644--- a/skills/metabase-data-app-semantic-layer/SKILL.md+++ b/skills/metabase-data-app-semantic-layer/SKILL.md@@ -11,7 +11,7 @@ metadata:Keep the semantic layer and presentation layer separate.-- All Metabase context must come from the generated schema file, usually `src/metabase.data.ts` or `src/*.metabase.data.ts` (its runtime object and the `/* metadata: {...} */` block an entry opens with when it has context to give), and what the files of the app's collection are written from must come from `npm run print-resources`. Never from Metabase YAML in the repository, and never from your own API calls.+- All Metabase context must come from the generated schema file, usually `src/metabase.data.ts` or `src/*.metabase.data.ts` (its runtime object and the `/* metadata: {...} */` block an entry opens with when it has context to give), and the files of the app's collection must come from `npm run write-resources`. Never from Metabase YAML in the repository, and never from your own API calls.- Do not discover data through MCP tools, create Metabase content, create tables, or edit the semantic layer while building the React UI.- Import data app query helpers from `@metabase/embedding-sdk-react/data-app`.- Every query is a `defineQuery(...)` named export in the root-level `queries/` directory, and every action a `defineAction(...)` named export in the root-level `actions/` directory, both beside `package.json`. Create both directories before writing the first hook call; the template ships them, each with a README. The hooks enforce this at compile time: `useMetabaseQuery`, `useMetabaseQueryObject`, and `useAction` reject an inline object, a `satisfies MetabaseQueryOptions` object, and a spread copy of a definition. The error reads `Property 'definedWithDefineQuery' is missing` (or `'definedWithDefineAction'`); the fix is always to move the object into `queries/` or `actions/` as a definition and import it, never a cast.@@ -47,32 +47,14 @@ Keep the semantic layer and presentation layer separate.## Generate Schema-If the schema file already exists, use it. If it is missing or stale, treat schema generation as semantic-layer curation for this data app, not a mechanical export.+If the schema file already exists and covers what the app needs, use it. Otherwise generate it.-Choose the export scope before generating:--1. Honor an explicit scope. Otherwise infer the required content types from the app's purpose: tables, curated metrics, or actions. For example, "show orders" needs tables; row counts and sums can also use table aggregations.-2. Choose the narrowest supported scope that covers those needs, using collection IDs and database names or IDs from the request or project context. When the library type is clear but a narrower collection is unknown, use that library's whole tree.-3. Ask only for context needed to select a scope, such as which database to use when the request requires a database scope but does not identify one. Once the scope is determined, state it briefly and generate without waiting for confirmation.--Scope parameters:--- `include-data-library=true` for the whole `Library / Data` tree.-- `include-metric-library=true` for the whole `Library / metrics` tree.-- `library-collections=<id-or-entity-id>[,<id-or-entity-id>]` for specific Data or metrics library subcollections.-- `include-actions=true` for the actions the app can run, under `schema.actions`. When combined with `database=<name-or-id>`, it includes that database's actions only.-- `database=<name-or-id>` when the app should use tables from one database. Use it separately from library scopes; the API rejects that combination.--Combine library scopes when the app needs both tables and curated metrics.--Use `include-actions=true` when the app needs any saved action under `schema.actions`; it includes all readable actions, unless `database` scopes them to one database. It can be combined with `library-collections`, `include-data-library`, or `include-metric-library` so one schema can include selected tables/metrics plus all readable actions.--If the user asks for any mutation-like flow, such as creating, updating, deleting, submitting, approving, executing an action, or running a write operation, include `include-actions=true` in the typed-schema URL. Do this even when the user names one specific action, because actions are only discoverable through the generated schema.+The schema always covers the same content: the published tables of the `Library / Data` tree, the metrics of the `Library / Metrics` tree (with the tables they map), and every query action that belongs to no model, under `schema.actions`. There is no scope to choose. When the app needs a table, metric, or action the schema lacks, ask the user to publish the table to the Data library, move the metric into the Metrics library, or create the action as a query action without a model, then regenerate.The Metabase URL and API key live in the **repo-root** `.env.local` as`DATA_APP_MB_URL` and `DATA_APP_MB_API_KEY` (one file per repo, usually two levels upfrom the app dir, not in the app dir). The key must be one in the Administrators group:-the typed schema answers only an admin, and any other key gets a `403`. The command below `source`s that file so+the schema answers only an admin, and any other key gets a `403`. The command below `source`s that file sothe shell substitutes the values straight into `curl` — you never read, extract,or handle the credentials yourself.@@ -83,9 +65,7 @@ or handle the credentials yourself.> or still set to the default `mb_replace_me` placeholder after sourcing, ask> the user to add real values themselves, then continue.-Source the credentials from the repo-root `.env.local` and generate the scoped-schema. The example below exports table data from the Data library; replace its-query parameters with the scope chosen above:+Source the credentials from the repo-root `.env.local` and generate the schema:```bashROOT="$(git rev-parse --show-toplevel 2>/dev/null)"@@ -106,13 +86,13 @@ fi-o src/metabase.data.ts \-H "x-api-key: $DATA_APP_MB_API_KEY" \-H "Accept: text/typescript" \- "$DATA_APP_MB_URL/api/typed-schemas/v1/typescript?include-data-library=true"+ "$DATA_APP_MB_URL/api/apps/generate/schemas")```An entry in the generated file opens with a `/* metadata: {...} */` block of JSON, before its runtime properties, when it has context to give for writing the app: such as a field's display name, description, and semantic type, a table's database, schema, and real name, and a metric's description, filters, and whether it's verified. It is a comment, so it never reaches the bundle and the app never imports it; read it, and never edit or reformat the file.-After a successful export, verify that the schema contains every entity needed for the requested app. If any are missing, revise the scope using available context or ask for the missing context before building the UI.+After a successful export, verify that the schema contains every entity needed for the requested app. If any are missing, tell the user what to publish to the library before building the UI.If schema generation fails, do not hide, paraphrase away, or retry past the error. Surface the typed-schema error to the user, including the failing ids, names, and message when present.@@ -137,7 +117,7 @@ export const CreateOrder = defineAction({});```-Each definition is backed by serialized Metabase YAML that you write into the repository's `collections/data_apps/` directory, where the collections of the `data-apps` namespace live, as Metabase exports them. Nothing changes in Metabase until the repository is pulled; the pull loads the files into the app's collection as serialized content, like everything else the repository holds, so write the files instead of creating anything in that collection by hand. From the app's directory, `data_apps/<slug>/`, that directory is `../../collections/data_apps/`.+Each definition is backed by serialized Metabase YAML that `npm run write-resources` writes into the repository's `collections/data_apps/` directory, where the collections of the `data-apps` namespace live, as Metabase exports them. Nothing changes in Metabase until the repository is pulled; the pull loads the files into the app's collection as serialized content, like everything else the repository holds, so write the files instead of creating anything in that collection by hand. From the app's directory, `data_apps/<slug>/`, that directory is `../../collections/data_apps/`.```data_apps/<slug>/data_app.yaml collection: <entity ID of the app's collection>@@ -147,34 +127,23 @@ collections/data_apps/<collection>/<name>.yaml a saved question per query, c`<collection>` is the collection's name as Metabase slugs it: `data_app__sales` for `Data App: Sales`. The pull and the CLI find the app's files by content, not by path: the collection whose `entity_id` the manifest names, and the cards and actions whose `collection_id` is it.-Write the YAML in the Metabase representation format. **Before writing or editing any file under `collections/data_apps/`, load the skill for reading and writing Metabase representation YAML** (use skill discovery), and follow it and its format spec for every file there: the collection, cards, and actions. If no such skill is available, install it with `npx skills add metabase/agent-skills/skills --skill metabase-representation-format`; if it still can't be loaded, stop and tell the user instead of writing the files. None of these replaces it: YAML from an earlier app (including files in git history), a search of the spec for one field, or `npm run check-resources` passing, whose schema validation checks the schema, not the format's conventions. Every card and copy is written from what `npm run print-resources` prints, as below; the format skill is how you write it. Never guess a field's shape. Generate every new entity ID with `npx representations generate-entity-id` (`--count <n>` for several), from the app's own dev dependencies; never invent one, reuse one from another app, or keep a source entity's ID on its copy.--1. **Collection.** Once per app, and before anything else under `collections/data_apps/`, write `collections/data_apps/<collection>.yaml` as a collection named `Data App: <app name>` with `namespace: data-apps`, and put its entity ID in `data_app.yaml` as `collection: <id>`. Scaffolding normally writes it with the manifest: skip this step when `data_app.yaml` already names a collection whose file exists. `print-resources` refuses to run until the manifest names the collection, since it prints the saved questions into it. It must be a root collection of that namespace: no `parent_id`, `personal_owner_id`, `type`, `authority_level`, or `archive_operation_id`, and not `is_remote_synced`, `is_sample`, or `archived`. The pull refuses a manifest without `collection:`, and one whose collection has no file in the repository.--2. **Print what the resources are written from.** Every `defineQuery` carries its `savedQuestionEntityId` and every `defineAction` its `copiedActionEntityId` before you print, generated with `npx representations generate-entity-id`: the saved question is printed with the definition's ID, so `print-resources` refuses a query without one and names it. Then `npm run print-resources` (optionally with one file, `npm run print-resources -- queries/revenue.query.ts`) sends the app's definitions to Metabase, using `DATA_APP_MB_URL` and `DATA_APP_MB_API_KEY` from the repo-root `.env.local`, and prints, as JSON, everything as serialization writes it, every reference by name or entity ID rather than numeric ID:- - `queries`: each definition's `export`, `file`, and `savedQuestionEntityId`, the saved question Metabase writes for it as `entity` (named after the export, in the app's collection, with that entity ID, holding the query the dev preview runs), and the entity IDs of the `metrics` it aggregates;- - `actions`: each definition's `export`, `file`, and `copiedActionEntityId`, and the source action as `entity`;- - `metrics`: the metrics the queries aggregate, each as `entity`.-- The numeric `id` beside an action or metric only says which source it is; no file of the app's collection holds it.-- The export answers only an admin, so the API key must be one in the Administrators group. With any other key the command fails with `403`: stop and tell the user, and never write the files another way.+Metabase writes every file of the app's collection: setting up the app writes the collection's own file, and `npm run write-resources` the rest. Never write or edit one yourself. Generate every new entity ID with `npx representations generate-entity-id` (`--count <n>` for several), from the app's own dev dependencies; never invent one, reuse one from another app, or keep a source entity's ID on its copy.- An item that can't be built or copied has an `error` instead (for example a field the query names that doesn't exist, an action that belongs to a model, or a metric or action that reads another card); stop and tell the user. Run it again after any definition changes, and update the files it affects.+1. **Collection.** Setting up the app writes the collection's file under `collections/data_apps/` and names it in `data_app.yaml` as `collection: <id>`, through `POST /api/apps/generate/app` (see the setup skill). The pull refuses a manifest without `collection:`, and one whose collection has no file in the repository.-3. **A saved question per query.** Write the printed `entity` to `collections/data_apps/<collection>/<name>.yaml` as it is. Its name, `type`, `display`, `creator_id`, `collection_id`, entity ID, and `serdes/meta` are already set; add nothing. The one change is in its `dataset_query`: each metric reference (`[metric, {}, <entity ID>]`) points at the app's copy of that metric (step 4) instead of the source metric's entity ID. Write it yourself in the format skill's layout (how it lays out MBQL clauses), keeping the printed key order, not by serializing the printed JSON with a script: a script skips the format skill, and its output isn't checked against the format. The query is what the dev preview ran, so any difference in it changes what production runs; after writing the file, parse it back and compare it with the printed entity, and fix every difference other than the metric references.+2. **Entity IDs.** Every `defineQuery` carries a `savedQuestionEntityId`, the entity ID of its saved question, and every `defineAction` a `copiedActionEntityId`, the entity ID of its copy, generated with `npx representations generate-entity-id`. The command refuses a definition without one and names it.-4. **A copy per metric.** Each metric a query aggregates must be copied too, since viewers can read only the app's collection. Write the metric's printed `entity` into `collections/data_apps/<collection>/` as a card with a new entity ID and `collection_id` set to the app's collection, and reference the copy's entity ID from every question that uses the metric. One copy serves them all.+3. **Write the files.** `npm run write-resources` sends the app's definitions to Metabase, using `DATA_APP_MB_URL` and `DATA_APP_MB_API_KEY` from the repo-root `.env.local`, and regenerates the app's collection folder (`collections/data_apps/<collection>/`): it deletes the cards and actions the collection held and writes what comes back, each file with the text a Metabase export writes. No pull is needed first. It writes: a saved question per query (named after the export, in the app's collection, with its `savedQuestionEntityId`, holding the query the dev preview runs), a copy of each action the app runs, and a copy of each metric a query aggregates, which the question already references. It prints a `Wrote <path>` line per file.-5. **A copy per action.** Write the action's printed `entity` into `collections/data_apps/<collection>/` with a new entity ID, set as the definition's `copiedActionEntityId`, and `collection_id` set to the app's collection. The app runs only query actions that belong to no model, so a copy never has a `model_id`.+ The command answers only an admin, so the API key must be one in the Administrators group. With any other key it fails with `403`: stop and tell the user, and never write the files another way.-Write every card and copy yourself from the printed export, following the format skill, never from YAML (not the repository's exported collections or its top-level `actions/`, not another app's collection, not git history), never from your own API calls, and never with a script that dumps the printed JSON as YAML. Its references are already in the form the YAML uses, so copy them as they are. A copy keeps exactly what its export prints, except what makes it a copy: a new `entity_id` (and the matching `serdes/meta` `id`, with a label from the copy's name) and `collection_id`. The print already leaves out what serialization leaves unset or at its default (`description`, `archived`, `enable_embedding`, `public_uuid`, a form field's unset settings, ...), as the format omits them: never add a key the print doesn't have, never keep one with a `null` value, and keep the printed key order.+ An item that can't be built or copied fails the whole command with its error, and nothing is written or deleted (for example a field the query names that doesn't exist, an action that belongs to a model, or a metric or action that reads another card); stop and tell the user. Run it again after any definition changes; it regenerates the folder.-Every card and action carries the `creator_id` the print gives; Metabase replaces it with its internal user when it loads the app.+Never write or edit a saved question, an action copy or a metric copy yourself, never copy one from YAML (not the repository's exported collections or its top-level `actions/`, not another app's collection, not git history), and never from your own API calls: a hand-written file differs from what Metabase serializes, and the query is what the dev preview ran, so any difference changes what production runs.-If a source the app needs isn't in `src/metabase.data.ts`, or its export has an `error`, stop and tell the user; never write a replacement from scratch. After a source changes in Metabase, run `npm run print-resources` again and write its copy again, keeping the copy's entity ID.+If a source the app needs isn't in `src/metabase.data.ts`, stop and tell the user; never write a replacement from scratch. After a source changes in Metabase, run `npm run write-resources` again.-Run `npm run check-resources`, and fix every file it reports. It fails, listing every problem, when the definitions and the files of the app's collection disagree: a manifest naming no collection or one without a file, a definition without its entity ID or naming one no file holds, a query naming a card that isn't a question, or a question or action no definition names; then it validates every Metabase YAML file in the repository, the app's collection files included, against the format's schema. `npm run build` runs the first check. Delete a metric copy yourself once nothing uses it. Neither command calls Metabase. The next pull validates everything again, and refuses the app's resources with a message naming the file, shown on the Data apps admin page. Fix that file and push again.+Run `npm run check-resources`, and fix what it reports by changing the definitions or `data_app.yaml` and running `npm run write-resources` again, never by editing a generated file. It fails, listing every problem, when the definitions and the files of the app's collection disagree: a manifest naming no collection or one without a file, a definition without its entity ID or naming one no file holds, a query naming a card that isn't a question, or a question or action no definition names; then it validates every Metabase YAML file in the repository, the app's collection files included, against the format's schema. `npm run build` runs the first check. `write-resources` removes a metric copy once nothing uses it. `check-resources` never calls Metabase. The next pull validates everything again, and refuses the app's resources with a message naming the file, shown on the Data apps admin page. Fix the definition or source it names, run `npm run write-resources`, and push again.Pass the definition itself to the hook and let the SDK resolve what runs — a production build runs the copy, while the dev preview runs the authored table or action, so an app works before its resources exist:@@ -188,7 +157,7 @@ const { execute, isExecuting, error } = useAction(CreateOrder);Never pass an inline table-source query (not even a read-only, filter-option, or helper query), a raw action id, `savedQuestionEntityId`, `copiedActionEntityId`, or a hand-built `{ source: { type: "card", id } }`, and never spread a definition into a new object. Each defeats the swap; the authored ids also bypass the permission boundary. TypeScript rejects most of these: the hooks accept only what `defineQuery`/`defineAction` returned, so an inline object, a `satisfies`-typed object, a spread copy, and `schema.actions.<action>` all fail to compile. When `tsc` reports `Property 'definedWithDefineQuery' is missing` or `Property 'definedWithDefineAction' is missing`, the argument is not a definition: move it into `queries/` or `actions/` and import the export. Do not silence it with a cast or by wrapping the inline object in `defineQuery(...)` at the call site, which compiles but is never backed by a saved question. Keep fixed permission-boundary filters, aggregations, and breakouts inside `defineQuery` — they are baked into the saved question, so don't apply them again outside it, and put runtime clauses in the hook's second argument (see *Static and dynamic query parts*). `useAction` needs no generics: the definition types `execute`'s parameters and `result`.-Wire `package.json` with `"print-resources": "embedding-sdk-react data-apps print-resources"`, `"check-resources": "embedding-sdk-react data-apps check-resources && representations validate-schema --folder ../.."` (with `@metabase/representations` as a dev dependency), and `"build": "vite build"`. After adding, changing, renaming, or removing any definition, update the files of the app's collection, then run `npm run check-resources` and `npm run build`. Do not test or hand off the app until `npm run build` succeeds. Commit `data_app.yaml`, the app's files under `collections/data_apps/`, and the definitions together; the app picks them up when Metabase pulls the repository.+Wire `package.json` with `"write-resources": "embedding-sdk-react data-apps write-resources"`, `"check-resources": "embedding-sdk-react data-apps check-resources && representations validate-schema --folder ../.."` (with `@metabase/representations` as a dev dependency), and `"build": "vite build"`. After adding, changing, renaming, or removing any definition, run `npm run write-resources`, then `npm run check-resources` and `npm run build`. Do not test or hand off the app until `npm run build` succeeds. Commit `data_app.yaml`, the app's files under `collections/data_apps/`, and the definitions together; the app picks them up when Metabase pulls the repository.## Standard pattern@@ -727,8 +696,7 @@ If no curated schema entry supports the intended UI, leave the section out or as- Run `npm run typecheck`. `Property 'definedWithDefineQuery' is missing` or `Property 'definedWithDefineAction' is missing` means a hook received something other than a `queries/` or `actions/` export; move the object there and import it.- Search touched files for `useMetabaseQuery(`, `useMetabaseQueryObject(`, and `useAction(`. The first argument must be an identifier imported from `queries/` or `actions/`; a `{`, a `defineQuery(`, a `defineAction(`, or a spread there is wrong even when it compiles. The second argument of the query hooks is the dynamic object and is written inline.- Confirm `queries/` and `actions/` sit beside `package.json`, not under `src/`, and that every definition the app renders lives there.-- Confirm you loaded the representation format skill and checked every file of the app's collection against it; say so in the hand-off, or say that you didn't.-- Confirm every definition has its entity ID and its file in the app's collection, and each saved question matches its definition. Run `npm run check-resources`, then `npm run build`; both fail when the app's collection files no longer back the definitions.+- Confirm every definition has its entity ID, and run `npm run write-resources`, then `npm run check-resources` and `npm run build`; both fail when the app's collection files no longer back the definitions.- Keep TypeScript diagnostics compact in the chat or handoff. Use the full output locally to fix the app, but report grouped root causes and only a few representative diagnostics instead of pasting the entire `tsc` output.- Verify every rendered value can be traced to a returned row property, schema field, measure, or deterministic transform.- Search touched files for `row[0]`, `row[1]`, `row.orderedAt`, `row.orderDate`, `as unknown as`, `DisplayRow`, `<select`, `margin`, `rate`, `score`, `percent`, `%`, `* 100`, and `.toFixed`; fix positional rows, result-key guesses, entity `<select>` filters, and unsupported business-field interpretations.@@ -738,9 +706,9 @@ If no curated schema entry supports the intended UI, leave the section out or as## Common Mistakes-- Creating or searching for Metabase content during app building. The files of the app's collection are written from `npm run print-resources`, never created in Metabase.-- Copying a metric or action from Metabase YAML in the repository (its exports, its top-level `actions/`, or git history), or from your own API calls, instead of from `npm run print-resources`.-- Writing a saved question yourself, adding a key the print doesn't have, or changing anything in the printed entity other than a metric reference to the app's copy.+- Creating or searching for Metabase content during app building. The files of the app's collection are written by `npm run write-resources`, never created in Metabase.+- Copying a metric or action from Metabase YAML in the repository (its exports, its top-level `actions/`, or git history), or from your own API calls, instead of letting `npm run write-resources` write it.+- Writing or editing a saved question, action copy or metric copy yourself instead of running `npm run write-resources`.- Writing the query object at the hook call instead of exporting it from `queries/` with `defineQuery`, or the action at `useAction` instead of from `actions/` with `defineAction`. Both are compile errors now; the fix is the directory, not a cast.- Wrapping the inline object in `defineQuery(...)` or `defineAction(...)` at the call site. It compiles, but it is never backed by a saved question, so it is refused in production.- Putting definitions under `src/queries/` or `src/actions/`, where the CLI never looks.diff --git a/skills/metabase-data-app-setup/SKILL.md b/skills/metabase-data-app-setup/SKILL.mdindex 0b28aa99f12..6ba4f89cec0 100644--- a/skills/metabase-data-app-setup/SKILL.md+++ b/skills/metabase-data-app-setup/SKILL.md@@ -53,9 +53,24 @@ If `<repo>/data_apps/<slug>/` already holds a project, verify it matches the cur2. `src/index.tsx` default-exports a `DataAppFactory` (type from`@metabase/embedding-sdk-react/data-app`) returning `{ component, providerProps? }`(no args).-3. `data_app.yaml` declares the same `version:` as this skill's- `template/data_app.yaml` (a manifest without the line is version 1). A lower- version is not drift but an outdated app: **Stop.** Migrating it is a+3. `data_app.yaml` declares the `version:` Metabase serves (a manifest without+ the line is version 1). Print the version Metabase serves without writing+ anything (never reuse the Step 4 snippet here: it writes files):++ ```bash+ ROOT="$(git rev-parse --show-toplevel 2>/dev/null)"+ (+ source "$ROOT/.env.local" 2>/dev/null+ curl -sS -X POST \+ -H "x-api-key: $DATA_APP_MB_API_KEY" \+ -H "Content-Type: application/json" \+ -d '{"name": "Version check"}' \+ "$DATA_APP_MB_URL/api/apps/generate/app" |+ node -e 'let s="";process.stdin.on("data",(d)=>(s+=d)).on("end",()=>{const r=JSON.parse(s);if(!r.files){console.error(s);process.exit(1)}console.log(r.files[0].yaml.match(/^version: (\d+)/m)[1])})'+ )+ ```++ A lower version is not drift but an outdated app: **Stop.** Migrating it is aseparate task; use the agent's normal skill-discovery flow for migrating anoutdated data app before extending it.@@ -130,23 +145,34 @@ Once the template is in `<repo>/data_apps/<slug>/` (run everything below from thIf it prints `MISSING`, **ask the user to fill `DATA_APP_MB_URL` (the running Metabase instance) and `DATA_APP_MB_API_KEY` (Admin → Authentication → API keys, a key in the Administrators group, since the typed schema answers only an admin) in `<repo>/.env.local` themselves** — up front, before anything needs the key.> **Never ask the user to paste the API key into the chat, and never `cat` / `echo` / print `.env.local` or its variables.** It's git-ignored and may hold *other* secrets — the file's contents and the key must never enter the conversation or your context. Every command that needs the key `source`s the file (as above) so the shell uses the value directly; you only ever see the `creds present` / `MISSING` signal, never the secret itself. (`creds present` only means both vars are filled and not the default `mb_replace_me` placeholder — not that the URL or key are valid; a bad key surfaces later when a request fails.)-5. `npm install` (or whichever package manager the user prefers — the template ships with no lockfile, so `npm` / `yarn` / `pnpm` / `bun` all work; use the project's existing lockfile if one appears post-clone).-6. **The lockfile and the built bundle must both be committed.** Metabase serves the file at the `path` declared in `data_app.yaml` (the template builds to `dist/index.js`) straight from the committed Git tree, and the lockfile keeps installs reproducible. **Verify with `git status`** after `npm install` and a build: both must appear as committable files.-7. `npm run dev` and confirm the preview at http://localhost:5174 renders the starter "Hello, data app" message.-8. If the preview hits CORS, add `http://localhost:5174` under Admin → Embedding → Embedded analytics SDK → CORS.-9. **Fill in `data_app.yaml`**, the template's manifest, now in the app directory. This is the per-app config Metabase reads on sync, one file per app. Its fields, which of them are required, and their rules are defined in the Data App section of the Metabase representation format spec: load the skill for reading and writing Metabase representation YAML (use skill discovery) and follow its spec. If no such skill is available, install it with `npx skills add metabase/agent-skills/skills --skill metabase-representation-format`; if it still can't be loaded, stop and tell the user. Set `name`, `slug`, and `description` for this app, and `entity_id` to an ID from `npx representations generate-entity-id` (from the app's own dev dependencies, after `npm install`). Leave `version` and `serdes/meta` as the template ships them, and `path` too unless you change the build output. `collection` stays empty until item 10. Never copy a value from another app's manifest.+5. **Generate `data_app.yaml` and the app's collection.** Metabase writes both, with new entity IDs: `POST /api/apps/generate/app` answers the app's `data_app.yaml` (at `data_apps/<slug>/data_app.yaml`) and its collection's file under `collections/data_apps/`, each at its path from the repo root. Run it from the repo, with the app's display name, its `/apps/<slug>` slug, and an optional one-line description:++ ```bash+ ROOT="$(git rev-parse --show-toplevel 2>/dev/null)"+ (+ source "$ROOT/.env.local" 2>/dev/null+ curl -sS -X POST \+ -H "x-api-key: $DATA_APP_MB_API_KEY" \+ -H "Content-Type: application/json" \+ -d '{"name": "Sales App", "slug": "sales-app", "description": "Pipeline health and quota attainment by region"}' \+ "$DATA_APP_MB_URL/api/apps/generate/app" |+ node -e 'let s="";process.stdin.on("data",(d)=>(s+=d)).on("end",()=>{const r=JSON.parse(s);if(!r.files){console.error(s);process.exit(1)}for(const f of r.files){const p=require("path").join(process.argv[1],f.path);require("fs").mkdirSync(require("path").dirname(p),{recursive:true});require("fs").writeFileSync(p,f.yaml);console.log("Wrote "+f.path)}})' "$ROOT"+ )+ ```++ Run it once per app: a second run writes a new collection with new entity IDs. Never copy `data_app.yaml` or its `collection:` line from another app. Edit the generated `data_app.yaml` only to add `allowed_hosts` (below).**`description`** — optional: a single short sentence saying what the appdoes, shown under its name in the admin UI so admins can tell apps apart at aglance. Sync folds any whitespace into single spaces and rejects anything over255 characters; the admin list wraps what is left rather than cutting it off,so a sentence reads well there and a paragraph crowds out the rows around it.- Replace the template's placeholder with a real sentence about *this* app, or- delete the line entirely if it adds nothing beyond the name.+ Pass a real sentence about *this* app, or leave `description` out of the+ request if it adds nothing beyond the name.**`version`** — the data app contract version this app's code targets, a- whole number. The template ships the version this skill targets; do not- change it by hand. Metabase bumps the version it serves only on a breaking+ whole number. Metabase writes the version it serves; do not change it by+ hand. Metabase bumps the version it serves only on a breakingchange to the contract. An app on an older version is marked *Outdated* inthe admin list, hidden from every other user, and refuses to open until itis migrated to the current contract, its `version` raised to match, rebuilt,@@ -170,27 +196,24 @@ Once the template is in `<repo>/data_apps/<slug>/` (run everything below from thmust also permit framing (`X-Frame-Options`/`frame-ancestors`) — many publicsites don't.-10. **Write the app's collection.** Generate its entity ID with- `npx representations generate-entity-id`, write the collection's file under- the repo's `collections/data_apps/` (use skill discovery to find the- data-app guidance on writing the files of an app's collection), and set that- ID as `collection` in `data_app.yaml`.+6. `npm install` (or whichever package manager the user prefers — the template ships with no lockfile, so `npm` / `yarn` / `pnpm` / `bun` all work; use the project's existing lockfile if one appears post-clone).+7. **The lockfile and the built bundle must both be committed.** Metabase serves the file at the `path` declared in `data_app.yaml` (the template builds to `dist/index.js`) straight from the committed Git tree, and the lockfile keeps installs reproducible. **Verify with `git status`** after `npm install` and a build: both must appear as committable files.+8. `npm run dev` and confirm the preview at http://localhost:5174 renders the starter "Hello, data app" message.+9. If the preview hits CORS, add `http://localhost:5174` under Admin → Embedding → Embedded analytics SDK → CORS.### The order of the app's filesAn app's YAML is written in this order, and each step needs the one before it:-1. `data_app.yaml`, copied from the template in Step 3 and filled in at item 9.-2. The collection's file under `collections/data_apps/`, named in- `data_app.yaml` as `collection` at item 10.-3. `npm run print-resources`, once the app has definitions in `queries/` or- `actions/`. It prints the saved questions into the collection- `data_app.yaml` names, so it refuses to run until the manifest names one.-4. The remaining files in the collection's directory: a saved question per- query and copies of the metrics and actions they use, written from what it- printed.--Scaffolding ends after the second. The last two are part of building the app's+1. `data_app.yaml` and the collection's file under `collections/data_apps/`,+ both generated at item 5.+2. The files in the collection's directory: a saved question per query and+ copies of the metrics and actions they use, which `npm run write-resources`+ regenerates once the app has definitions in `queries/` or `actions/`. It+ writes into the collection `data_app.yaml` names, so it refuses to run until+ the manifest names one whose file exists.++Scaffolding ends after the first. The second is part of building the app'sdata layer; use skill discovery for that.## Step 5 — Verify the starter app@@ -544,8 +567,8 @@ Do not wrap `InteractiveQuestion` or `StaticQuestion` in containers that clip orData apps are delivered by Git — you commit the app directory and Metabase pulls it on its next remote-sync import.-1. After any change to `queries/` or `actions/`, update the files of the app's collection under `collections/data_apps/` to match (regenerating `src/metabase.data.ts` first if it doesn't cover what the definitions use), and run `npm run check-resources`. Nothing reaches Metabase until the pull in step 4 applies it.-2. `npm run build` produces the bundle at your `data_app.yaml` `path` (the template builds to `dist/index.js`). It fails when the app's collection files don't match the definitions; fix them and rebuild.+1. After any change to `queries/` or `actions/`, run `npm run write-resources` (regenerating `src/metabase.data.ts` first if it doesn't cover what the definitions use), then `npm run check-resources`. Nothing reaches Metabase until the pull in step 4 applies it.+2. `npm run build` produces the bundle at your `data_app.yaml` `path` (the template builds to `dist/index.js`). It fails when the app's collection files don't match the definitions; run `npm run write-resources` and rebuild.3. From the **repo root**, commit the app directory — its `data_app.yaml`, the built bundle (the file `path` points at), the source, and the lockfile — together with the app's collection files, and **push**:```bashgit add data_apps/<slug> collections/data_apps@@ -556,7 +579,7 @@ Data apps are delivered by Git — you commit the app directory and Metabase pul> **Don't offer to "deploy" the app or ask how the bundle reaches a staging environment** — there is no separate deploy step, and the question only confuses users: Metabase imports the committed bundle straight from the connected repo on its next sync. Once the change is on the branch Metabase syncs from — however the user gets it there (a merged PR, or a push straight to that branch) — just tell them to pull it in and open the app in Metabase at `/apps/<slug>`.-- **To update:** update the app's collection files if the definitions changed, commit a new build, and pull again.+- **To update:** run `npm run write-resources` if the definitions changed, commit a new build, and pull again.- **To remove:** see *Removing an app* below.## Removing an appdiff --git a/skills/metabase-data-app-setup/template/README.md b/skills/metabase-data-app-setup/template/README.mdindex e2e20cf26bb..db225f13abe 100644--- a/skills/metabase-data-app-setup/template/README.md+++ b/skills/metabase-data-app-setup/template/README.md@@ -10,11 +10,11 @@ coding agent and the Metabase data-app skills.```bashnpm run dev # preview at http://localhost:5174, in the sandbox Metabase runs apps innpm run typecheck # type-checks src/ and vite.config.ts-npm run print-resources # prints what the app's collection files are written from, as JSON+npm run write-resources # has Metabase regenerate the app's collection files under collections/data_apps/npm run check-resources # checks the app's collection files against its definitions, then validates the repository's Metabase YAMLnpm run build # builds dist/index.js, the bundle data_app.yaml's path names```-`npm run dev` and `npm run print-resources` read `DATA_APP_MB_URL` and+`npm run dev` and `npm run write-resources` read `DATA_APP_MB_URL` and`DATA_APP_MB_API_KEY` from the `.env.local` at the repository root;`.env.local.example` lists them.diff --git a/skills/metabase-data-app-setup/template/actions/README.md b/skills/metabase-data-app-setup/template/actions/README.mdindex 65d0877b1bb..107e5c1f2ee 100644--- a/skills/metabase-data-app-setup/template/actions/README.md+++ b/skills/metabase-data-app-setup/template/actions/README.md@@ -55,14 +55,15 @@ Rules:- This directory sits beside `package.json`, not under `src/`. The CLI scansonly `queries/` and `actions/`, so a definition anywhere else never gets acopy, and the authored action is refused for the app's viewers in production.-- Actions exist only when the generated schema includes actions- (`include-actions=true`). The app runs a copy of each action, written from- what `npm run print-resources` prints; it never creates actions.+- The generated schema lists every query action that belongs to no model.+ The app runs a copy of each action, written by+ `npm run write-resources`; it never creates actions.- Pass the export itself to `useAction`. Never pass`schema.actions.<action>` or its `.id`.- `copiedActionEntityId` is the entity ID of the action's copy in the app'scollection, under the repo's `collections/data_apps/`. After adding a- definition, write the copy there, run `npm run check-resources`, and commit+ definition, run `npm run write-resources` to write the copy there, then+ `npm run check-resources`, and committhe definitions and the collection files together. `npm run build` failsuntil they match.- Never copy a `copiedActionEntityId` to another definition, or remove it whilediff --git a/skills/metabase-data-app-setup/template/data_app.yaml b/skills/metabase-data-app-setup/template/data_app.yamldeleted file mode 100644index fed07487e26..00000000000--- a/skills/metabase-data-app-setup/template/data_app.yaml+++ /dev/null@@ -1,26 +0,0 @@-version: 1--name: Data App Template--slug: data-app-template--description: A summary of what this app does--path: ./dist/index.js--# Origins this app may call with fetch/XHR. Everything else is blocked — both in-# `npm run dev` (via the dev server's CSP) and inside Metabase (the sandbox).-# The Metabase instance itself is reached through the SDK, not listed here.-# allowed_hosts:-# - https://api.example.com-# - https://*.internal.acme.com--# The entity ID of the app's collection. Write the collection's file under the-# repo's `collections/data_apps/` first, then set its ID here.-collection:--# Generate with `npx representations generate-entity-id`.-entity_id:--serdes/meta:-- model: DataAppdiff --git a/skills/metabase-data-app-setup/template/package.json b/skills/metabase-data-app-setup/template/package.jsonindex f839ea2c657..26a7f09a530 100644--- a/skills/metabase-data-app-setup/template/package.json+++ b/skills/metabase-data-app-setup/template/package.json@@ -6,7 +6,7 @@"scripts": {"dev": "vite --port 5174","build": "vite build",- "print-resources": "embedding-sdk-react data-apps print-resources",+ "write-resources": "embedding-sdk-react data-apps write-resources","check-resources": "embedding-sdk-react data-apps check-resources && representations validate-schema --folder ../..","typecheck": "tsc --noEmit"},diff --git a/skills/metabase-data-app-setup/template/queries/README.md b/skills/metabase-data-app-setup/template/queries/README.mdindex ee1267b070e..f6a91b43602 100644--- a/skills/metabase-data-app-setup/template/queries/README.md+++ b/skills/metabase-data-app-setup/template/queries/README.md@@ -69,10 +69,9 @@ Rules:- Pass the export itself to the hook. Never spread or copy it.- Each definition carries `savedQuestionEntityId`, the entity ID of its savedquestion in the app's collection, under the repo's `collections/data_apps/`.- After adding or changing a definition, update that card- (`npm run print-resources` shows what it must hold). A metric it aggregates- is copied into the app's collection too, written from what the same command- prints. Then run `npm run check-resources`, and commit the definitions and+ After adding or changing a definition, run `npm run write-resources` to+ write that card. A metric it aggregates is copied into the app's collection+ too, by the same command. Then run `npm run check-resources`, and commit the definitions andthe collection files together. `npm run build` fails until they match.- Never copy a `savedQuestionEntityId` to another definition, or remove it whileits card exists.diff --git a/src/metabase/api_routes/routes.clj b/src/metabase/api_routes/routes.cljindex 24804e59e2c..01cda63777b 100644--- a/src/metabase/api_routes/routes.clj+++ b/src/metabase/api_routes/routes.clj@@ -71,7 +71,6 @@[metabase.transforms-rest.api.transform][metabase.transforms-rest.api.transform-job][metabase.transforms-rest.api.transform-tag]- [metabase.typed-schemas-rest.api][metabase.upload.api][metabase.user-key-value.api][metabase.users-rest.api]@@ -127,7 +126,6 @@metabase.transforms-rest.api.transform/keep-memetabase.transforms-rest.api.transform-job/keep-memetabase.transforms-rest.api.transform-tag/keep-me- metabase.typed-schemas-rest.api/keep-memetabase.upload.api/keep-memetabase.user-key-value.api/keep-memetabase.users-rest.api/keep-me@@ -275,7 +273,6 @@"/transform-dag-run" (+auth metabase.transforms-rest.api.transform/transform-dag-run-routes)"/transform-job" (+auth metabase.transforms-rest.api.transform/transform-job-routes)"/transform-tag" (+auth metabase.transforms-rest.api.transform/transform-tag-routes)- "/typed-schemas" (+auth 'metabase.typed-schemas-rest.api)"/upload" (+auth 'metabase.upload.api)"/user" (+auth 'metabase.users-rest.api)"/user-key-value" (+auth 'metabase.user-key-value.api)diff --git a/src/metabase/typed_schemas/build.clj b/src/metabase/typed_schemas/build.cljdeleted file mode 100644index e89a24ad656..00000000000--- a/src/metabase/typed_schemas/build.clj+++ /dev/null@@ -1,147 +0,0 @@-(ns metabase.typed-schemas.build- "Builds the semantic schema value: interprets [[SemanticSchemaOptions]] into- SchemaSource reads and assembles the fetched items into the document.-- Internal to the typed-schemas module: the public surface is- [[metabase.typed-schemas.core]], which composes [[fetch-items]] and- [[create-schema]] into its public functions. See the core namespace- docstring for the pipeline shape and the separation rules."- (:require- [clojure.set :as set]- [metabase.system.core :as system]- [metabase.typed-schemas.common :as common]- [metabase.typed-schemas.source :as source]- [metabase.util.malli.registry :as mr]- [metabase.util.malli.schema :as ms])- (:import- (java.time Instant)))--(set! *warn-on-reflection* true)--(def ^:private DatabaseRef- "References a database by id or name."- [:or- [:map [:id :int]]- ; remote sync representations expose database names instead of numeric database ids- [:map [:name ms/NonBlankString]]])--(def ^:private CollectionRef- "References a collection by numeric or entity id."- [:or- [:map [:id :int]]- [:map [:entity-id ms/NonBlankString]]])--(def SemanticSchemaOptions- "Schema generation options accepted by [[fetch-items]]."- [:map {:closed true}- [:database {:optional true}- [:maybe {:description (str "Scopes the schema to a database. "- "Accepts `{:id <database-id>}` or `{:name <database-name>}`.")}- DatabaseRef]]- [:library-collection-refs {:optional true}- [:sequential {:description (str "Limits tables and metrics to library collections. "- "Each reference accepts `{:id <collection-id>}` or `{:entity-id <collection-entity-id>}`.")}- CollectionRef]]- [:include-data-library? {:optional true}- [:boolean {:description "Whether to include the root data library."}]]- [:include-metric-library? {:optional true}- [:boolean {:description "Whether to include the root metrics library."}]]- [:include-actions? {:optional true}- [:boolean {:description (str "Include query actions that belong to no model. Database scope filters them; "- "library scope does not. Without a scope, returns actions only.")}]]])--(def Items- "Fetched schema entities, ready for pure assembly by [[create-schema]].-- Each entry holds entity maps shaped by the `metabase.typed-schemas.schema.*`- builders; `:key` on each entity seeds the generated object keys."- [:map {:closed true}- [:actions [:sequential :map]]- [:tables [:sequential :map]]- [:metrics [:sequential :map]]])--(defn- invalid-options!- [message]- (throw (ex-info message {:status-code 400})))--(defn- validate-options!- [{:keys [database library-collection-refs- include-data-library? include-metric-library?] :as options}]- (when-not (mr/validate SemanticSchemaOptions options)- (invalid-options! "Invalid semantic schema options."))- (let [include-library-root? (or include-data-library? include-metric-library?)- collection-scoped? (or (seq library-collection-refs)- include-library-root?)]- (when (and collection-scoped? database)- (invalid-options!- "Collection-scoped query parameters and database query parameters are mutually exclusive."))))--(defn- library-items- "Returns tables and metrics for a library scope.-- Tables mapped by in-scope metrics are included even when they live outside- the library, so rendered metric dimensions can reference their fields."- [source library-scope]- (let [metrics (source/metrics source nil (:metric-collection-ids library-scope))- table-ids (set/union (into #{} (map :id) (source/library-tables source library-scope))- (into #{} (mapcat :mappedTableIds) metrics))]- {:tables (source/tables source nil table-ids)- :metrics metrics}))--(defn fetch-items- "Fetches the schema entities selected by [[SemanticSchemaOptions]].-- This is the only impure stage of the pipeline: everything downstream of the- returned [[Items]] is pure. All reads go through a- [[metabase.typed-schemas.source/SchemaSource]] — the application database by- default; tests reify the protocol with literal values."- ([options]- (fetch-items options source/app-db-source))- ([options source]- (let [{:keys [database library-collection-refs- include-data-library? include-metric-library? include-actions?]- :or {library-collection-refs []- include-data-library? false- include-metric-library? false- include-actions? false}} options]- (validate-options! (assoc options- :library-collection-refs library-collection-refs- :include-data-library? include-data-library?- :include-metric-library? include-metric-library?- :include-actions? include-actions?))- (let [library-scope (source/library-scope source- {:library-collection-refs library-collection-refs- :include-data-library? include-data-library?- :include-metric-library? include-metric-library?})- database-ids (source/database-ids source database)- action-schemas (if include-actions?- ;; database-ids scopes the actions; nil reads all of them- (source/actions source database-ids)- [])]- (if (or library-scope- (and include-actions? (nil? database-ids)))- (let [{:keys [tables metrics]} (when library-scope- (library-items source library-scope))]- {:actions (vec action-schemas)- :tables (vec tables)- :metrics (vec metrics)})- {:actions (vec action-schemas)- :tables (source/tables source database-ids nil)- :metrics (source/metrics source database-ids nil)})))))--(defn create-schema- "Assembles fetched [[Items]] into the semantic schema value rendered by- [[render-typescript]].-- Pure when `info` provides `:generated-at` and `:instance-url`; they default- to the current time and the configured site URL."- ([items]- (create-schema items nil))- ([{:keys [actions tables metrics]} {:keys [generated-at instance-url]}]- (array-map- :schemaVersion 2- :generatedAt (str (or generated-at (Instant/now)))- :metabase {:instanceUrl (or instance-url (system/site-url))}- :actions (common/keyed-map actions)- :tables (common/keyed-map tables)- :metrics (common/keyed-map metrics))))diff --git a/src/metabase/typed_schemas/core.clj b/src/metabase/typed_schemas/core.cljdeleted file mode 100644index da8d89d275f..00000000000--- a/src/metabase/typed_schemas/core.clj+++ /dev/null@@ -1,64 +0,0 @@-(ns metabase.typed-schemas.core- "Typed schemas: TypeScript modules that describe questions, actions, tables- and metrics to coding agents. Runtime objects- feed the Lib.createTestQuery DSL; each entry's `/* metadata: {...} */` block- carries context for humans and agents.-- The public surface is deliberately small — the schema value and its- rendering, mirroring the module's data/print split:-- (-> options build-semantic-schema render-typescript) ; options -> TS string-- These public functions are compositions of the internal pipeline stages;- their definitions below show the whole pipeline. The schema is for- superusers, so fetching no longer filters by what the caller can read. The- table and metric details lookups still read-check the current user, so a- non-admin calling [[build-semantic-schema]] gets a 403, and callers outside- a request must bind a current-user context first.-- Keep the separation when extending this module:-- - New data reads go through [[metabase.typed-schemas.source/SchemaSource]] —- a protocol method plus its `app-db-source` implementation — never direct- `t2` calls from the pipeline stages.- - Everything downstream of fetched items stays pure. Timestamps, site URL,- and any other environment values enter through the `info` argument, not by- calling out from assembly or rendering code.- - Which keys render as runtime data vs metadata is policy in- `metabase.typed-schemas.render`; TypeScript syntax lives only in the- `metabase.typed-schemas.javascript` printer.- - REST query-parameter strings are decoded once, in- `metabase.typed-schemas-rest.api.query-params`; this module only ever sees- typed [[SemanticSchemaOptions]].- - New exports here are compatibility promises, and they need not map 1:1 to- internal functions — expose what consumers need, keep the stages internal."- (:require- [metabase.typed-schemas.build :as build]- [metabase.typed-schemas.javascript :as javascript]- [metabase.typed-schemas.render :as render]))--(set! *warn-on-reflection* true)--(def SemanticSchemaOptions- "Schema generation options accepted by [[build-semantic-schema]]."- build/SemanticSchemaOptions)--(defn build-semantic-schema- "Builds a semantic schema value from [[SemanticSchemaOptions]].-- `info` optionally pins `:generated-at` and `:instance-url` for deterministic- output; they default to the current time and the configured site URL."- ([options]- (build-semantic-schema options nil))- ([options info]- (-> options- build/fetch-items ; all data access, returns items- (build/create-schema info)))) ; pure assembly into the schema value--(defn render-typescript- "Renders a semantic schema value as an ES module of `as const` TypeScript- constants."- [schema]- (-> schema- render/schema->ast ; pure render policy, returns a TypeScript AST- javascript/render-js)) ; pure, option-free printerdiff --git a/src/metabase/typed_schemas/db.clj b/src/metabase/typed_schemas/db.cljdeleted file mode 100644index 51a0d6225f2..00000000000--- a/src/metabase/typed_schemas/db.clj+++ /dev/null@@ -1,132 +0,0 @@-(ns metabase.typed-schemas.db- "Application database queries for the typed schemas module. Every function here is a direct Toucan 2 call with no- additional logic, so the rest of the module never talks to `toucan2.core` itself."- (:require- [metabase.lib.schema.id :as lib.schema.id]- [metabase.queries.core :as queries]- [metabase.util.malli :as mu]- [metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]- [toucan2.core :as t2]))--(mu/defn destination-database-ids- "The ids among `database-ids` of Databases that are routing destinations."- [database-ids :- [:set ::lib.schema.id/database]]- (t2/select-fn-set :id :model/Database :id [:in database-ids] :router_database_id [:not= nil]))--(defn- scope-filter-clause- "Compiles a resolved scope into a Honey SQL where-clause conjunct: a nil scope means unscoped (no clause), an- empty scope matches nothing."- [scope-ids column]- (when scope-ids- (if (seq scope-ids)- [:in column scope-ids]- ;; no row has id -1: a resolved-but-empty scope matches no rows- [:= column -1])))--(mu/defn cards-ordered-by-name- "The non-archived Cards of `card-type` among `database-ids` and/or `collection-ids` (either nil for unscoped), in- name then id order. The schema is for superusers, who see every collection, and what an archived collection or- the trash holds is archived itself."- [card-type :- [:enum :model :question :metric]- database-ids :- [:maybe [:set ::lib.schema.id/database]]- collection-ids :- [:maybe [:set ::lib.schema.id/collection]]]- (t2/select :model/Card- {:where [:and- [:= :type (name card-type)]- [:= :archived false]- (scope-filter-clause database-ids :database_id)- (scope-filter-clause collection-ids :collection_id)]- :order-by [[:name :asc] [:id :asc]]}))--(mu/defn field-ids-and-table-ids- "The id and Table id of the Fields with `field-ids`."- [field-ids :- [:set ::lib.schema.id/field]]- (t2/select [:model/Field :id :table_id] :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))--(mu/defn card-dimensions- "The query-relevant columns (`:dimensions` and `:dimension_mappings` among them) of the Card with `card-id`,- or nil."- [card-id :- ::lib.schema.id/card]- (queries/card-query-info card-id))--(mu/defn table-names- "The id, database id, name, and display name of the Tables with `table-ids`."- [table-ids :- [:sequential ::lib.schema.id/table]]- (t2/select [:model/Table :id :db_id :name :display_name] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query)]}))--(mu/defn model-less-query-action-ids- "The ids of the unarchived query Actions without a model outside `excluded-collection-ids`, among `database-ids`- (nil for unscoped), in name then id order."- [database-ids :- [:maybe [:set ::lib.schema.id/database]]- excluded-collection-ids :- [:set ::lib.schema.id/collection]]- (t2/select-pks-vec :model/Action- {:where [:and- [:= :model_id nil]- [:= :type "query"]- [:= :archived false]- (when (seq excluded-collection-ids)- [:or [:= :collection_id nil] [:not-in :collection_id excluded-collection-ids]])- (when database-ids- [:exists ^:allow-subquery {:select [1]- :from [[(t2/table-name :model/QueryAction) :qa]]- :where [:and- [:= :qa.action_id :action.id]- (scope-filter-clause database-ids :qa.database_id)]}])]- :order-by [[:name :asc] [:id :asc]]}))--(mu/defn field-table-id- "The Table id of the Field with `field-id`, or nil."- [field-id :- ::lib.schema.id/field]- (t2/select-one-fn :table_id :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))--(mu/defn active-tables-in-scope- "The active Tables among `database-ids` and/or `table-ids` (either nil for unscoped), in name then id order."- [database-ids :- [:maybe [:set ::lib.schema.id/database]]- table-ids :- [:maybe [:or [:set ::lib.schema.id/table] [:sequential ::lib.schema.id/table]]]]- (t2/select :model/Table- {:from [(warehouse-schema-overlay/table-query)]- :where [:and [:= :active true]- (scope-filter-clause database-ids :db_id)- (scope-filter-clause table-ids :id)]- :order-by [[:name :asc] [:id :asc]]}))--(mu/defn published-library-tables-in-collections- "The active, published Tables in `collection-ids` (nil for unscoped), in name then id order."- [collection-ids :- [:maybe [:set ::lib.schema.id/collection]]]- (t2/select :model/Table- {:from [(warehouse-schema-overlay/table-query)]- :where [:and- [:= :active true]- [:= :is_published true]- (scope-filter-clause collection-ids :collection_id)]- :order-by [[:name :asc] [:id :asc]]}))--(mu/defn measure-definition- "The definition of the Measure with `measure-id`, or nil."- [measure-id :- ::lib.schema.id/measure]- (t2/select-one-fn :definition :model/Measure :id measure-id))--(mu/defn measure-definitions- "The id and definition of the Measures with `measure-ids`."- [measure-ids :- [:sequential ::lib.schema.id/measure]]- (t2/select [:model/Measure :id :definition] :id [:in measure-ids]))--(mu/defn database- "The Database with `database-id`, or nil."- [database-id :- [:maybe ::lib.schema.id/database]]- (t2/select-one :model/Database :id database-id))--(mu/defn databases-named- "The Databases named `database-name`."- [database-name :- :string]- (t2/select :model/Database :name database-name))--(mu/defn collections- "The Collections with `collection-ids`."- [collection-ids :- [:set ::lib.schema.id/collection]]- (t2/select :model/Collection :id [:in collection-ids]))--(mu/defn collections-by-entity-ids- "The Collections with `entity-ids`."- [entity-ids :- [:set :string]]- (t2/select :model/Collection :entity_id [:in entity-ids]))diff --git a/src/metabase/typed_schemas/schema/common.clj b/src/metabase/typed_schemas/schema/common.cljdeleted file mode 100644index 60b8b844696..00000000000--- a/src/metabase/typed_schemas/schema/common.clj+++ /dev/null@@ -1,67 +0,0 @@-(ns metabase.typed-schemas.schema.common- "Shared typed-schema source helpers."- (:require- [medley.core :as m]- [metabase.audit-app.core :as audit]- [metabase.lib-be.core :as lib-be]- [metabase.lib.core :as lib]- [metabase.metabot.core :as metabot]- [metabase.premium-features.core :as premium-features]- [metabase.typed-schemas.db :as typed-schemas.db]))--(set! *warn-on-reflection* true)--(defn destination-db-ids- "Returns the subset of `db-ids` that back a destination (routed) database.-- Destinations are routing internals reachable only through their router database, so typed-schema- generation excludes anything backed by one -- the same rule Metabot's resource guard enforces (see- `metabase.metabot.tools.resources/check-resource-database`)."- [db-ids]- (when (seq db-ids)- (typed-schemas.db/destination-database-ids db-ids)))--(defn- without-unavailable-cards- "`cards` without any backed by a destination (routed) database -- see [[destination-db-ids]] -- and, while the- audit feature is off, without the audit database's and the audit collection's: the card details lookup refuses- one in the audit collection, and one on the audit database would be listed without its table."- [cards]- (let [destination-ids (destination-db-ids (into #{} (keep :database_id) cards))- audit-off? (not (premium-features/enable-audit-app?))]- (cond->> cards- (seq destination-ids) (remove #(contains? destination-ids (:database_id %)))- audit-off? (remove #(or (= audit/audit-db-id (:database_id %))- (some-> (:collection_id %) audit/is-collection-id-audit?))))))--(defn select-schema-cards- "Returns non-archived cards for schema generation.-- Metrics, models and saved questions are backed by cards. They need the same archived, database and collection- filters, and leave out the cards the schema can't serve -- see [[without-unavailable-cards]]."- [card-type database-ids collection-ids]- (without-unavailable-cards (typed-schemas.db/cards-ordered-by-name card-type database-ids collection-ids)))--(defn aggregation-result-column-with-metadata-provider- "Returns an aggregation result column using an existing metadata provider."- [metadata-provider query-definition]- (try- (let [query (lib/query metadata-provider query-definition)- aggregation-column (m/find-first #(= (:lib/source %) :source/aggregations)- (lib/returned-columns query))]- (when aggregation-column- (metabot/->result-column query aggregation-column)))- ;; Result-column inference is best effort; callers fall back to an unknown column.- (catch Exception _- nil)))--(defn aggregation-result-column- "Returns the first aggregation result column for a saved query- definition, for metrics and measures."- [database-id query-definition]- (try- (aggregation-result-column-with-metadata-provider- (lib-be/application-database-metadata-provider database-id)- query-definition)- ;; Result-column inference is best effort; callers fall back to an unknown column.- (catch Exception _- nil)))diff --git a/src/metabase/typed_schemas/schema/question.clj b/src/metabase/typed_schemas/schema/question.cljdeleted file mode 100644index 09a995da7d2..00000000000--- a/src/metabase/typed_schemas/schema/question.clj+++ /dev/null@@ -1,71 +0,0 @@-(ns metabase.typed-schemas.schema.question- "Typed schema generation for saved questions."- (:require- [medley.core :as m]- [metabase.metabot.tools.entity-details :as entity-details]- [metabase.typed-schemas.common :as common]- [metabase.typed-schemas.schema.common :as schema.common]))--(set! *warn-on-reflection* true)--(defn- question-details-error-message- [card error-message]- (format "Failed to build schema details for question \"%s\" (card %s): %s"- (or (:name card) "Untitled")- (:id card)- (or error-message "unknown error")))--(defn- question-details-error-data- [card error-data]- (m/assoc-some- {:card-id (:id card)- :card-name (:name card)- :card-type (:type card)}- :status-code (:status-code error-data)))--(defn- question-details- "Returns saved-question details in the selected-card order."- [cards]- (let [details-by-card-id (into {}- (mapcat (fn [[database-id database-cards]]- (mapv (fn [card details]- (try- [(:id card)- (assoc details- :result-columns (:fields details)- :display (:display card))]- (catch Exception exception- (throw (ex-info (question-details-error-message card (ex-message exception))- (assoc (question-details-error-data card (ex-data exception))- :cause-message (ex-message exception))- exception)))))- database-cards- (entity-details/cards-details- :question database-id database-cards- {:with-field-values? false- :with-related-tables? false- :with-metrics? false- :with-measures? false- :with-segments? false})))- (group-by :database_id cards)))]- (mapv details-by-card-id (map :id cards))))--(defn question-schema- "Returns the schema for a saved question."- [{:keys [id name description verified display result-columns portable_entity_id]}]- (m/assoc-some- {:type "card"- :key (common/generated-key name id)- :id id- :name name- :display display- :columns (mapv common/column-schema result-columns)}- :entityId portable_entity_id- :description description- :verified (when verified true)))--(defn question-schemas- "Returns schemas for saved questions, with optional database and collection scopes."- [database-ids collection-ids]- (let [cards (schema.common/select-schema-cards :question database-ids collection-ids)]- (mapv question-schema (question-details cards))))diff --git a/src/metabase/typed_schemas/scope.clj b/src/metabase/typed_schemas/scope.cljdeleted file mode 100644index cbf3a4fa83d..00000000000--- a/src/metabase/typed_schemas/scope.clj+++ /dev/null@@ -1,148 +0,0 @@-(ns metabase.typed-schemas.scope- "Resolves portable references into scopes.-- Callers hold *references*: portable ways to name content that survive- moving between instances — a database by id or name, a collection by id or- entity id, or a well-known library root by its fixed entity id. Remote-sync- representations and coding agents pass names and entity ids precisely- because numeric ids don't travel.-- A *scope* is what references resolve to on this instance: the concrete- sets of row ids that bound what fetching selects. The schema is for- superusers, so nothing is checked against the caller. Resolution:-- - expands collection references to include their descendants;- - distinguishes nil (no reference given: unscoped) from the empty set- (references resolved to nothing: match nothing);- - throws a 404 naming the collection references that don't resolve, while- database references that don't resolve yield an empty scope, and- therefore an empty schema rather than an error.-- Database and collection scopes are bare id sets; the library scope is a- [[LibraryScope]] map classifying the in-scope library tree by collection- type. Resolution is batched: query count is constant regardless of how many- references are passed."- (:require- [clojure.string :as str]- [metabase.collections.models.collection :as collection]- [metabase.typed-schemas.db :as typed-schemas.db]- [metabase.util.malli :as mu]- [metabase.util.malli.schema :as ms]))--(set! *warn-on-reflection* true)--(def ^:private library-data-entity-id- "Entity id of the root data library collection."- "librarylibrarydatadat")--(def ^:private library-metrics-entity-id- "Entity id of the root metrics library collection."- "librarylibrarymetrics")--(defn database-ids-for-ref- "Returns the database ids matching a typed database reference.-- An id matches at most one database. Database names are not unique, so a- name reference can genuinely match several databases — currently their- content is unioned into one scope."- [database-ref]- (when database-ref- (let [{:keys [id name]} database-ref]- (if id- (if (typed-schemas.db/database id) #{id} #{})- (into #{} (map :id) (typed-schemas.db/databases-named name))))))--(defn- not-found!- [collection-refs]- (throw (ex-info (str "Collections not found: "- (str/join ", " (map pr-str collection-refs)))- {:status-code 404- :collection-refs (vec collection-refs)})))--(defn- resolve-collection-refs!- "Batch-resolves collection references into collection rows, in ref order.-- Throws a 404 naming every reference that does not resolve to a collection- satisfying `usable-collection?`."- [collection-refs usable-collection?]- (let [ids (into #{} (keep :id) collection-refs)- entity-ids (into #{} (keep :entity-id) collection-refs)- usable? (fn [collection]- (and collection- (usable-collection? collection)))- by-id (when (seq ids)- (into {} (map (juxt :id identity))- (typed-schemas.db/collections ids)))- ;; entity_id is a fixed-width char column; some app dbs return it- ;; space-padded, so key the lookup by the trimmed value.- by-eid (when (seq entity-ids)- (into {} (map (juxt (comp str/trimr :entity_id) identity))- (typed-schemas.db/collections-by-entity-ids entity-ids)))- resolved (for [{:keys [id entity-id] :as collection-ref} collection-refs]- [collection-ref- (let [collection (if id (get by-id id) (get by-eid entity-id))]- (when (usable? collection)- collection))])- missing (into [] (comp (remove second) (map first)) resolved)]- (when (seq missing)- (not-found! missing))- (mapv second resolved)))--(defn collection-scope- "Returns ids for the referenced collections and their descendants."- [collection-refs]- (when (seq collection-refs)- (let [collections (resolve-collection-refs! collection-refs (constantly true))]- (into (into #{} (map :id) collections)- (map :id)- (collection/descendants-flat-for collections)))))--(def LibraryScope- "A resolved library scope: the library collection tree, expanded to- descendants and classified by collection type. `:data-collection-ids` bound- published table selection; `:metric-collection-ids` bound metric selection."- [:map {:closed true}- [:data-collection-ids [:set :int]]- [:metric-collection-ids [:set :int]]])--(defn- library-collection?- [collection]- (contains? collection/library-collection-types (:type collection)))--(def ^:private LibraryCollectionRef- "References a collection by numeric or entity id."- [:or- [:map {:closed true} [:id :int]]- [:map {:closed true} [:entity-id ms/NonBlankString]]])--(def LibraryScopeOptions- "Options accepted by [[library-scope]]."- [:map {:closed true}- [:library-collection-refs {:optional true} [:maybe [:sequential LibraryCollectionRef]]]- [:include-data-library? {:optional true} [:maybe :boolean]]- [:include-metric-library? {:optional true} [:maybe :boolean]]])--(defn- library-refs- "Returns collection references for explicit library refs plus any included- well-known library roots."- [{:keys [library-collection-refs include-data-library? include-metric-library?]}]- (concat library-collection-refs- (when include-data-library?- [{:entity-id library-data-entity-id}])- (when include-metric-library?- [{:entity-id library-metrics-entity-id}])))--(mu/defn library-scope :- [:maybe LibraryScope]- "Resolves library collection refs and include flags into a [[LibraryScope]],- or nil when no library scope is requested."- [scope-options :- LibraryScopeOptions]- (when-let [refs (seq (library-refs scope-options))]- (let [roots (resolve-collection-refs! refs library-collection?)- collections (concat roots (collection/descendants-flat-for roots))- ids-of-type (fn [collection-type]- (into #{}- (comp (filter #(= (:type %) collection-type))- (map :id))- collections))]- {:data-collection-ids (ids-of-type collection/library-data-collection-type)- :metric-collection-ids (ids-of-type collection/library-metrics-collection-type)})))diff --git a/src/metabase/typed_schemas/source.clj b/src/metabase/typed_schemas/source.cljdeleted file mode 100644index 4a8fc913341..00000000000--- a/src/metabase/typed_schemas/source.clj+++ /dev/null@@ -1,77 +0,0 @@-(ns metabase.typed-schemas.source- "Data access for typed schemas, reified as a protocol.-- [[SchemaSource]] names every read that [[metabase.typed-schemas.build/fetch-items]]- performs, so the module's data-access surface is one definition instead of- selects scattered across namespaces. [[app-db-source]] is the production- implementation, backed by the application database.-- Tests reify the protocol with literal values instead of redefining selection- functions:-- (reify source/SchemaSource- (library-scope [_ _] {:metric-collection-ids #{20}})- (metrics [_ _ _] [{:type \"metric\" :key \"revenue\" :id 1}])- ...)-- All methods return the shaped schema entities produced by the- `metabase.typed-schemas.schema.*` builders, except [[library-tables]], which- returns raw table rows (only `:id` is consumed).-- When the pipeline needs to read something new, add a protocol method and its- [[app-db-source]] implementation here — do not call `t2`/`metabot` directly- from `metabase.typed-schemas.build` or anything downstream of it. That keeps- the module's data-access surface enumerable and every downstream stage- testable with literal values."- (:require- [metabase.typed-schemas.schema.action :as schema.action]- [metabase.typed-schemas.schema.metric :as schema.metric]- [metabase.typed-schemas.schema.question :as schema.question]- [metabase.typed-schemas.schema.table :as schema.table]- [metabase.typed-schemas.scope :as scope]))--(set! *warn-on-reflection* true)--(defprotocol SchemaSource- "The data-access patterns behind typed schema generation.-- Scope-resolution methods return ids; entity methods return shaped schema- entities. A nil `database-ids`/`collection-ids` argument means unscoped;- an empty set matches nothing."- (database-ids [source database-ref]- "Database ids matching a database reference, or nil without one.")- (collection-ids [source collection-refs]- "Ids of the referenced collections and their descendants, or nil without refs.")- (library-scope [source scope-options]- "Resolved [[metabase.typed-schemas.scope/LibraryScope]] for library- collection refs and include flags, or nil when none are requested.")- (questions [source database-ids collection-ids]- "Question schema entities.")- (actions [source database-ids]- "Schema entities for query actions that belong to no model.")- (metrics [source database-ids collection-ids]- "Metric schema entities.")- (tables [source database-ids table-ids]- "Table schema entities.")- (library-tables [source library-scope]- "Published table rows in the library scope's data collections."))--(def app-db-source- "The production [[SchemaSource]], backed by the application database."- (reify SchemaSource- (database-ids [_ database-ref]- (scope/database-ids-for-ref database-ref))- (collection-ids [_ collection-refs]- (scope/collection-scope collection-refs))- (library-scope [_ scope-options]- (scope/library-scope scope-options))- (questions [_ database-ids collection-ids]- (vec (schema.question/question-schemas database-ids collection-ids)))- (actions [_ database-ids]- (schema.action/action-schemas database-ids))- (metrics [_ database-ids collection-ids]- (vec (schema.metric/metric-schemas database-ids collection-ids)))- (tables [_ database-ids table-ids]- (vec (schema.table/table-schemas (schema.table/select-tables database-ids table-ids))))- (library-tables [_ library-scope]- (vec (schema.table/select-library-tables library-scope)))))diff --git a/src/metabase/typed_schemas_rest/api.clj b/src/metabase/typed_schemas_rest/api.cljdeleted file mode 100644index 0c9abce5ed1..00000000000--- a/src/metabase/typed_schemas_rest/api.clj+++ /dev/null@@ -1,63 +0,0 @@-(ns metabase.typed-schemas-rest.api- "/api/typed-schemas endpoints.-- This is the only place REST-shaped strings exist: query parameters decode to- typed options in [[metabase.typed-schemas-rest.api.query-params]] at this- boundary, and everything in `metabase.typed-schemas.*` works on typed data."- (:require- [metabase.api.common :as api]- [metabase.api.macros :as api.macros]- [metabase.typed-schemas-rest.api.query-params :as query-params]- [metabase.typed-schemas.core :as typed-schemas]- [metabase.util.malli.schema :as ms]))--(set! *warn-on-reflection* true)--(def ^:private typescript-response-headers- {"Content-Type" "text/typescript; charset=utf-8"- "X-Content-Type-Options" "nosniff"- "Cross-Origin-Resource-Policy" "same-origin"- "Referrer-Policy" "no-referrer"- "Cache-Control" "no-store"})--(def ^:private TypedSchemaQueryParams- [:map {:closed true}- [:database {:optional true}- [:maybe {:description "Scopes the schema to a database by numeric id or name."}- ms/NonBlankString]]- [:library-collections {:optional true}- [:maybe {:description (str "Comma-separated library collection ids or entity ids. "- "Limits tables and metrics to those library collections.")}- ms/NonBlankString]]- [:include-data-library {:optional true}- [:maybe {:description "Whether to include the entire data library."}- :boolean]]- [:include-metric-library {:optional true}- [:maybe {:description "Whether to include the entire metric library."}- :boolean]]- [:include-actions {:optional true}- [:maybe {:description (str "Include query actions that belong to no model. Database scope filters them; "- "library scope does not. Without a scope, returns actions only.")}- :boolean]]])--(api.macros/defendpoint :get "/v1/typescript" :- :any- "Generate a TypeScript semantic schema module. For superusers: a data app's author builds from it, and only an- admin works with an app's repository."- [_route-params- query-params :- TypedSchemaQueryParams- _body-params- {{question-collections "question-collections"- include-models "include-models"} :query-params}]- (api/check-superuser)- (when (some? question-collections)- (throw (ex-info "The question-collections query parameter is not supported."- {:status-code 400})))- (when (some? include-models)- (throw (ex-info "The include-models query parameter is not supported; use include-actions."- {:status-code 400})))- {:status 200- :headers typescript-response-headers- :body (-> query-params- query-params/query-params->options- typed-schemas/build-semantic-schema- typed-schemas/render-typescript)})diff --git a/src/metabase/typed_schemas_rest/api/query_params.clj b/src/metabase/typed_schemas_rest/api/query_params.cljdeleted file mode 100644index 3acf382c4a4..00000000000--- a/src/metabase/typed_schemas_rest/api/query_params.clj+++ /dev/null@@ -1,45 +0,0 @@-(ns metabase.typed-schemas-rest.api.query-params- "REST query parameter parsing for typed schemas."- (:require- [clojure.string :as str]))--(set! *warn-on-reflection* true)--(defn- non-blank-string- [value]- (some-> value str str/trim not-empty))--(defn- database-reference- [value]- (when-let [value (non-blank-string value)]- (if-let [id (parse-long value)]- {:id id}- {:name value})))--(defn- comma-separated-references- [value]- (if-let [value (non-blank-string value)]- (->> (str/split value #",")- (keep (fn [value]- (when-let [value (non-blank-string value)]- (if-let [id (parse-long value)]- {:id id}- {:entity-id value}))))- vec)- []))--(defn- enabled?- [value]- (contains? #{true "true" "1"} value))--(defn query-params->options- "Converts REST query parameters into semantic schema options.-- This is the sole typed-schema function that accepts REST-shaped query- parameters."- [query-params]- {:database (database-reference (:database query-params))- :library-collection-refs (comma-separated-references (:library-collections query-params))- :include-data-library? (enabled? (:include-data-library query-params))- :include-metric-library? (enabled? (:include-metric-library query-params))- :include-actions? (enabled? (:include-actions query-params))})diff --git a/test/metabase/metabot/tools/entity_details_test.clj b/test/metabase/metabot/tools/entity_details_test.cljindex 11a10ca04cb..11194e2a451 100644--- a/test/metabase/metabot/tools/entity_details_test.clj+++ b/test/metabase/metabot/tools/entity_details_test.clj@@ -603,7 +603,7 @@:dataset_query (mt/mbql-query venues{:aggregation [[:count]]:breakout [$price]})}]- ;; Exercise `cards-details`, the batch path used by the typed-schemas endpoint. Calling+ ;; Exercise `cards-details`, the batch path used by the data app schema endpoint. Calling;; `get-table-details` for each Card would miss a failure that terminates the complete sequence.(let [details (->> (entity-details/cards-details :question (mt/id) [broken good] {})(into [] (map #(select-keys % [:name :query_json]))))]diff --git a/test/metabase/typed_schemas/core_test.clj b/test/metabase/typed_schemas/core_test.cljdeleted file mode 100644index ca50cc8af79..00000000000--- a/test/metabase/typed_schemas/core_test.clj+++ /dev/null@@ -1,190 +0,0 @@-(ns metabase.typed-schemas.core-test- "Typed schema tests form a pyramid; put new tests at the lowest level that- can express them:-- - In-memory (no db): printer goldens in `javascript-test` (the output format- spec), structural =? AST tests in `render-test`, pure shaping in- `common-test` and the `schema.*` tests, assembly from literal [[Items]]- and scoping rules against a reified literal source here.- - Db seam (with-temp): `source-test` for `app-db-source`, `scope-test` for- library-tree classification.- - One end-to-end test here on `mt/dataset test-data`, through the whole- pipeline to rendered TypeScript."- (:require- [clojure.string :as str]- [clojure.test :refer :all]- [metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata]- [metabase.test :as mt]- [metabase.test.fixtures :as fixtures]- [metabase.typed-schemas.build :as build]- [metabase.typed-schemas.core :as typed-schemas]- [metabase.typed-schemas.source :as source]))--(use-fixtures :once (fixtures/initialize :db :test-users))--(def ^:private test-info- {:generated-at "2026-01-01T00:00:00Z"- :instance-url "https://metabase.example.com"})--(deftest options-reject-unknown-keys-test- (is (thrown-with-msg? clojure.lang.ExceptionInfo- #"Invalid semantic schema options\."- (build/fetch-items {:unknown-option true}))))--(deftest options-reject-collection-and-database-scopes-together-test- (is (thrown-with-msg? clojure.lang.ExceptionInfo- #"mutually exclusive"- (build/fetch-items {:database {:id 1}- :include-data-library? true}))))--(deftest create-schema-assembles-items-test- (is (= {:schemaVersion 2- :generatedAt "2026-01-01T00:00:00Z"- :metabase {:instanceUrl "https://metabase.example.com"}- :actions {"shipOrder" {:kind "action", :key "shipOrder", :id 11}}- :tables {"orders" {:type "table", :key "orders", :id 3}}- :metrics {"revenue" {:type "metric", :key "revenue", :id 2}}}- (build/create-schema- {:actions [{:kind "action", :key "shipOrder", :id 11}]- :tables [{:type "table", :key "orders", :id 3}]- :metrics [{:type "metric", :key "revenue", :id 2}]}- test-info))))--(deftest create-schema-disambiguates-duplicate-keys-test- (let [schema (build/create-schema- {:actions []- :tables [{:type "table", :key "orders", :id 3}- {:type "table", :key "orders", :id 4}]- :metrics []}- test-info)]- (is (= ["orders3" "orders4"] (keys (:tables schema))))- (is (= ["orders3" "orders4"] (map :key (vals (:tables schema)))))))--(deftest create-schema-defaults-info-test- (let [schema (build/create-schema {:actions [], :tables [], :metrics []})]- (is (string? (:generatedAt schema)))- (is (contains? (:metabase schema) :instanceUrl))))--(defn- literal-source- "A [[source/SchemaSource]] over literal values. `tables` are filtered by the- requested table ids so tests can see which tables fetching asked for."- [{:keys [database-ids collection-ids library-scope library-tables- questions actions metrics tables]}]- (reify source/SchemaSource- (database-ids [_ _] database-ids)- (collection-ids [_ _] collection-ids)- (library-scope [_ _] library-scope)- (questions [_ _ _] (vec questions))- (actions [_ _] (vec actions))- (metrics [_ _ _] (vec metrics))- (tables [_ _ table-ids] (cond->> (vec tables)- table-ids (filterv #(contains? table-ids (:id %)))))- (library-tables [_ _] (vec library-tables))))--;; fetch-items decides *what* to fetch; the source supplies literal rows, so-;; the test pins the scoping rules, not the db.-(deftest fetch-items-includes-tables-mapped-by-library-metrics-test- (let [source (literal-source- {:library-scope {:metric-collection-ids #{20}- :data-collection-ids #{10}}- :library-tables [{:id 10}]- :metrics [{:type "metric", :key "revenue", :id 1, :mappedTableIds [42]}]- :tables [{:id 10, :type "table", :key "publishedTable"}- {:id 42, :type "table", :key "mappedTable"}- {:id 99, :type "table", :key "notInScope"}]})]- (is (= {:actions []- :tables [{:id 10, :type "table", :key "publishedTable"}- {:id 42, :type "table", :key "mappedTable"}]- :metrics [{:type "metric", :key "revenue", :id 1, :mappedTableIds [42]}]}- (build/fetch-items {:include-data-library? true} source)))))--(deftest options-reject-question-collection-refs-test- (is (thrown-with-msg? clojure.lang.ExceptionInfo- #"Invalid semantic schema options\."- (build/fetch-items {:question-collection-refs [{:id 1}]}))))--(deftest fetch-items-includes-actions-only-when-requested-test- (let [action-calls (atom [])- action {:kind "action", :key "shipOrder", :id 11}- source (reify source/SchemaSource- (database-ids [_ database-ref] (when database-ref #{26}))- (collection-ids [_ _] nil)- (library-scope [_ {:keys [include-data-library?]}]- (when include-data-library? {:data-collection-ids #{10}}))- (questions [_ _ _] [])- (actions [_ database-ids]- (swap! action-calls conj database-ids)- [action])- (metrics [_ _ _] [])- (tables [_ _ _] [])- (library-tables [_ _] []))]- (testing "no action is read unless asked for"- (is (=? {:actions []} (build/fetch-items {:database {:id 26}} source)))- (is (empty? @action-calls)))- (testing "a database scope scopes the actions to it"- (is (=? {:actions [action]} (build/fetch-items {:database {:id 26}, :include-actions? true} source))))- (testing "a library scope reads every readable action"- (is (=? {:actions [action]} (build/fetch-items {:include-data-library? true, :include-actions? true} source))))- (is (= [#{26} nil] @action-calls))))--;; One end-to-end test over the real test-data dataset: cards for every entity-;; kind on real synced tables, run through the whole pipeline to TypeScript.-;; Everything above tests the stages with cheap literal data; this proves the-;; composition works against the real thing. With real fields the question and-;; metric cards need no result_metadata — columns are computed from metadata.-(deftest full-pipeline-end-to-end-test- (mt/dataset test-data- ;; Use a copy of the Database so Cards committed concurrently by other namespaces cannot affect the- ;; database-scoped assertion below.- (mt/with-temp-copy-of-db- (mt/with-actions-enabled- (let [mp (mt/metadata-provider)- orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))- revenue-query (lib/aggregate orders-query- (lib/sum (lib.metadata/field mp (mt/id :orders :total))))]- (mt/with-temp [:model/Card _question {:name "Order totals", :database_id (mt/id), :table_id (mt/id :orders)- :type :question, :display :table- :dataset_query orders-query}- :model/Card _metric {:name "Order revenue", :database_id (mt/id), :table_id (mt/id :orders)- :type :metric, :display :scalar- :dataset_query revenue-query}- :model/Card model {:name "Order model", :database_id (mt/id), :table_id (mt/id :orders)- :type :model- :dataset_query orders-query- :result_metadata [{:name "total", :display_name "Total"- :base_type :type/Float- :field_ref [:field (mt/id :orders :total) nil]- :id (mt/id :orders :total)}]}- :model/Action action {:name "Update order", :model_id (:id model), :type :implicit}- :model/ImplicitAction _ {:action_id (:id action), :kind "row/update"}- :model/Action standalone {:name "Discount order", :type :query}- :model/QueryAction _ {:action_id (:id standalone)- :dataset_query (lib/native-query mp "UPDATE orders SET discount = 0 WHERE id = {{id}}")}]- (mt/with-current-user (mt/user->id :crowberto)- (let [schema (typed-schemas/build-semantic-schema {:database {:id (mt/id)}- :include-actions? true}- test-info)- body (typed-schemas/render-typescript schema)]- (testing "every entity kind lands in the schema with its real relationships"- (is (=? {:generatedAt "2026-01-01T00:00:00Z"- :metabase {:instanceUrl "https://metabase.example.com"}- :tables {"orders" {:fields {"total" {:jsType "number"}}}}- :metrics {"orderRevenue" {:mappedTableIds [(mt/id :orders)]- :columns [{:displayName "Sum of Total"- :jsType "number"}]}}- :actions {"discountOrder" {:kind "action", :id (:id standalone), :type "query"}}}- schema)))- (testing "saved questions are absent from the schema"- (is (not (contains? schema :questions))))- (testing "an action that belongs to a model stays out"- (is (not (str/includes? body "updateOrder"))))- (testing "only the temp metric and action are in scope for the dataset database"- (is (= {:metrics ["orderRevenue"], :actions ["discountOrder"]}- (update-vals (select-keys schema [:metrics :actions])- (comp vec keys)))))- (testing "the rendered module carries the real entities"- (is (str/includes? body "orders: {"))- (is (str/includes? body "name: \"Order revenue\""))- (is (str/includes? body "discountOrder: {"))- (is (str/ends-with? body "export default schema;\n")))))))))))diff --git a/test/metabase/typed_schemas/schema/question_test.clj b/test/metabase/typed_schemas/schema/question_test.cljdeleted file mode 100644index b560936a1e6..00000000000--- a/test/metabase/typed_schemas/schema/question_test.clj+++ /dev/null@@ -1,39 +0,0 @@-(ns metabase.typed-schemas.schema.question-test- (:require- [clojure.test :refer :all]- [metabase.metabot.tools.entity-details :as entity-details]- [metabase.test :as mt]- [metabase.typed-schemas.schema.common :as schema.common]- [metabase.typed-schemas.schema.question :as schema.question]))--(deftest question-schema-uses-card-source-discriminator-test- (is (= {:type "card"- :key "ordersQuestion"- :id 41- :name "Orders question"- :display "table"- :columns [{:type "column", :name "count", :displayName "Count", :jsType "number"}]}- (schema.question/question-schema- {:id 41- :name "Orders question"- :display "table"- :result-columns [{:name "count", :display_name "Count", :type :number}]}))))--(deftest question-schemas-bulk-loads-details-by-database-test- (let [cards [{:id 41 :name "Orders question" :type :question :database_id 1 :display "table"}- {:id 42 :name "People question" :type :question :database_id 1 :display "table"}]- bulk-details-calls (atom [])- report-details-calls (atom [])]- (mt/with-dynamic-fn-redefs [entity-details/get-report-details- (fn [{:keys [report-id] :as options}]- (swap! report-details-calls conj options)- {:structured-output {:id report-id :result-columns []}})]- (mt/with-dynamic-fn-redefs [schema.common/select-schema-cards (constantly cards)- entity-details/cards-details (fn [card-type database-id selected-cards _options]- (swap! bulk-details-calls conj [card-type database-id selected-cards])- (map #(assoc % :fields []) selected-cards))]- (is (= [41 42]- (mapv :id (schema.question/question-schemas nil nil))))- (is (= [[:question 1 cards]]- @bulk-details-calls))- (is (empty? @report-details-calls))))))diff --git a/test/metabase/typed_schemas/scope_test.clj b/test/metabase/typed_schemas/scope_test.cljdeleted file mode 100644index 434d8699799..00000000000--- a/test/metabase/typed_schemas/scope_test.clj+++ /dev/null@@ -1,131 +0,0 @@-(ns metabase.typed-schemas.scope-test- (:require- [clojure.test :refer :all]- [metabase.collections.models.collection :as collection]- [metabase.test :as mt]- [metabase.test.fixtures :as fixtures]- [metabase.typed-schemas.scope :as typed-schemas.scope]))--(use-fixtures :once (fixtures/initialize :db :test-users))--(deftest library-scope-accepts-comma-separated-subcollection-ids-test- (mt/with-temp [:model/Collection root {:name "Library"- :type "library"- :location "/"}- :model/Collection data {:name "Data"- :type "library-data"- :location (collection/children-location root)}- :model/Collection metrics {:name "Metrics"- :type "library-metrics"- :location (collection/children-location root)}- :model/Collection data-child {:name "Boba Data"- :type "library-data"- :location (collection/children-location data)}- :model/Collection data-grandkid {:name "Boba Data Nested"- :type "library-data"- :location (collection/children-location data-child)}- :model/Collection metric-child {:name "Boba Metrics"- :type "library-metrics"- :location (collection/children-location metrics)}]- (mt/with-test-user :crowberto- (is (=? {:data-collection-ids #{(:id data-child) (:id data-grandkid)}- :metric-collection-ids #{(:id metric-child)}}- (typed-schemas.scope/library-scope- {:library-collection-refs [{:id (:id data-child)} {:id (:id metric-child)}]}))))))--(deftest library-scope-accepts-representation-entity-ids-test- (mt/with-temp [:model/Collection root {:name "Library"- :type "library"- :location "/"}- :model/Collection data {:name "Data"- :type "library-data"- :location (collection/children-location root)}- :model/Collection website {:name "Website"- :type "library-data"- :entity_id "g-jLnamuHKdezZMthJ-z7"- :location (collection/children-location data)}- :model/Collection website-page {:name "Website Page"- :type "library-data"- :location (collection/children-location website)}]- (mt/with-test-user :crowberto- (is (=? {:data-collection-ids #{(:id website) (:id website-page)}- :metric-collection-ids #{}}- (typed-schemas.scope/library-scope- {:library-collection-refs [{:entity-id "g-jLnamuHKdezZMthJ-z7"}]}))))))--(deftest library-scope-includes-canonical-data-and-metrics-libraries-test- (with-redefs [typed-schemas.scope/library-data-entity-id "test-library-data"- typed-schemas.scope/library-metrics-entity-id "test-library-metrics"]- (mt/with-temp [:model/Collection root {:name "Library"- :type "library"- :location "/"}- :model/Collection data {:name "Data"- :type "library-data"- :entity_id "test-library-data"- :location (collection/children-location root)}- :model/Collection metrics {:name "Metrics"- :type "library-metrics"- :entity_id "test-library-metrics"- :location (collection/children-location root)}- :model/Collection data-child {:name "Boba Data"- :type "library-data"- :location (collection/children-location data)}- :model/Collection metric-child {:name "Boba Metrics"- :type "library-metrics"- :location (collection/children-location metrics)}]- (mt/with-test-user :crowberto- (is (=? {:data-collection-ids #{(:id data) (:id data-child)}- :metric-collection-ids #{(:id metrics) (:id metric-child)}}- (typed-schemas.scope/library-scope- {:include-data-library? true- :include-metric-library? true})))))))--(deftest library-scope-combines-explicit-and-root-collections-test- (with-redefs [typed-schemas.scope/library-metrics-entity-id "test-library-metrics"]- (mt/with-temp [:model/Collection root {:name "Library"- :type "library"- :location "/"}- :model/Collection data {:name "Data"- :type "library-data"- :location (collection/children-location root)}- :model/Collection metrics {:name "Metrics"- :type "library-metrics"- :entity_id "test-library-metrics"- :location (collection/children-location root)}- :model/Collection metric-child {:name "Boba Metrics"- :type "library-metrics"- :location (collection/children-location metrics)}]- (mt/with-test-user :crowberto- (is (=? {:data-collection-ids #{(:id data)}- :metric-collection-ids #{(:id metrics) (:id metric-child)}}- (typed-schemas.scope/library-scope- {:library-collection-refs [{:id (:id data)}]- :include-metric-library? true})))))))--(deftest question-collection-scope-accepts-comma-separated-collection-ids-test- (mt/with-temp [:model/Collection parent {:name "Question Parent"- :location "/"}- :model/Collection child {:name "Question Child"- :location (collection/children-location parent)}]- (mt/with-test-user :crowberto- (is (= #{(:id parent) (:id child)}- (typed-schemas.scope/collection-scope [{:id (:id parent)}]))))))--(deftest question-collection-scope-accepts-representation-entity-ids-test- (mt/with-temp [:model/Collection parent {:name "Question Parent"- :entity_id "question-entity-id-1"- :location "/"}- :model/Collection child {:name "Question Child"- :location (collection/children-location parent)}]- (mt/with-test-user :crowberto- (is (= #{(:id parent) (:id child)}- (typed-schemas.scope/collection-scope [{:entity-id "question-entity-id-1"}]))))))--(deftest question-collection-scope-rejects-missing-collection-ref-test- (mt/with-test-user :crowberto- (let [e (is (thrown-with-msg? clojure.lang.ExceptionInfo- #"Collections not found: \{:entity-id \"missing-entity-id-1\"\}"- (typed-schemas.scope/collection-scope [{:entity-id "missing-entity-id-1"}])))]- (is (=? {:status-code 404- :collection-refs [{:entity-id "missing-entity-id-1"}]}- (ex-data e))))))diff --git a/test/metabase/typed_schemas/source_test.clj b/test/metabase/typed_schemas/source_test.cljdeleted file mode 100644index 73a24396bee..00000000000--- a/test/metabase/typed_schemas/source_test.clj+++ /dev/null@@ -1,142 +0,0 @@-(ns metabase.typed-schemas.source-test- "Tests [[source/app-db-source]] against real application-database rows.-- Everything downstream of the source works on plain data and is covered by- cheap in-memory tests; these tests are the seam where typed schemas meet the- database."- (:require- [clojure.set :as set]- [clojure.test :refer :all]- [metabase.collections.models.collection :as collection]- [metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata]- [metabase.test :as mt]- [metabase.test.fixtures :as fixtures]- [metabase.typed-schemas.scope :as scope]- [metabase.typed-schemas.source :as source]- [metabase.util.malli.registry :as mr]))--(use-fixtures :once (fixtures/initialize :db :test-users))--(deftest database-ids-test- (mt/with-temp [:model/Database db {:name "Typed Schema Source DB"}]- (mt/with-current-user (mt/user->id :crowberto)- (testing "resolves a database by id and by name"- (is (= #{(:id db)} (source/database-ids source/app-db-source {:id (:id db)})))- (is (= #{(:id db)} (source/database-ids source/app-db-source {:name "Typed Schema Source DB"}))))- (testing "an unmatched reference resolves to an empty scope, not nil"- (is (= #{} (source/database-ids source/app-db-source {:name "__no_such_database__"}))))- (testing "no reference means unscoped"- (is (nil? (source/database-ids source/app-db-source nil)))))))--(deftest collection-ids-include-descendants-test- (mt/with-temp [:model/Collection parent {:name "Parent"}- :model/Collection child {:name "Child", :location (format "/%d/" (:id parent))}- :model/Collection grandchild {:name "Grandchild"- :location (format "/%d/%d/" (:id parent) (:id child))}]- (mt/with-current-user (mt/user->id :crowberto)- (testing "resolves numeric and entity-id references with their descendants"- (is (= #{(:id parent) (:id child) (:id grandchild)}- (source/collection-ids source/app-db-source [{:id (:id parent)}])))- (is (= #{(:id child) (:id grandchild)}- (source/collection-ids source/app-db-source [{:entity-id (:entity_id child)}]))))- (testing "no references means unscoped"- (is (nil? (source/collection-ids source/app-db-source [])))))))--(deftest library-scope-and-library-tables-test- (mt/with-temp [:model/Collection data-collection {:name "Data Library", :type "library-data"}- :model/Collection plain-collection {:name "Not A Library"}- :model/Database db {}- :model/Table published {:db_id (:id db), :name "published_table", :active true- :is_published true, :collection_id (:id data-collection)}- :model/Table _unpublished {:db_id (:id db), :name "unpublished_table", :active true- :is_published false, :collection_id (:id data-collection)}]- (mt/with-current-user (mt/user->id :crowberto)- (let [scope (source/library-scope source/app-db-source- {:library-collection-refs [{:id (:id data-collection)}]})]- (testing "library scope classifies collection ids by type"- (is (mr/validate scope/LibraryScope scope))- (is (= {:data-collection-ids #{(:id data-collection)}- :metric-collection-ids #{}}- scope)))- (testing "library tables are the published tables in the data collections"- (is (= [(:id published)]- (map :id (source/library-tables source/app-db-source scope))))))- (testing "a non-library collection reference is not found"- (is (thrown-with-msg? clojure.lang.ExceptionInfo- #"Collections not found"- (source/library-scope source/app-db-source- {:library-collection-refs [{:id (:id plain-collection)}]})))))))--(deftest metrics-honour-library-collection-scope-test- (mt/with-temp [:model/Collection root {:name "Library", :type "library", :location "/"}- :model/Collection metrics {:name "Metrics"- :type "library-metrics"- :location (collection/children-location root)}- :model/Collection child {:name "Child"- :type "library-metrics"- :location (collection/children-location metrics)}- :model/Card _metric {:name "Library revenue"- :type :metric- :collection_id (:id child)}]- (mt/with-test-user :crowberto- (let [{:keys [metric-collection-ids]}- (source/library-scope source/app-db-source- {:library-collection-refs [{:id (:id metrics)}]})]- (is (= ["libraryRevenue"]- (map :key- (source/metrics source/app-db-source nil metric-collection-ids))))- (is (= []- (source/metrics source/app-db-source nil #{})))))))--(deftest tables-test- (mt/with-temp [:model/Database db {}- :model/Table table {:db_id (:id db), :name "widgets", :display_name "Widgets", :active true}- :model/Field _ {:table_id (:id table), :name "price", :base_type :type/Float}]- (mt/with-current-user (mt/user->id :crowberto)- (testing "returns shaped table entities for a database scope"- (is (=? [{:type "table"- :key "widgets"- :fields {"price" {:jsType "number"}}}]- (source/tables source/app-db-source #{(:id db)} nil))))- (testing "an empty table-id scope matches nothing"- (is (= [] (source/tables source/app-db-source nil #{})))))))--(defn- touch-category-query []- (let [query (lib/native-query (mt/metadata-provider) "UPDATE categories SET name = name WHERE id = {{id}}")]- (lib/with-template-tags query (mapv #(assoc % :type :number) (lib/template-tags query)))))--(def ^:private id-parameter- {:id "id", :slug "id", :type :number/=, :target [:variable [:template-tag "id"]], :required true})--(deftest actions-test- (mt/with-temp [:model/Database {other-db-id :id} {}- :model/Collection {collection-id :id} {:name "Actions" :namespace "data-actions"}- :model/Card {model-id :id} {:type :model- :dataset_query (lib/query (mt/metadata-provider)- (lib.metadata/table (mt/metadata-provider)- (mt/id :categories)))}- :model/Action {standalone :id} {:type :query, :name "Touch category", :parameters [id-parameter]}- :model/QueryAction _ {:action_id standalone, :dataset_query (touch-category-query)}- :model/Action {archived :id} {:type :query, :name "Archived", :archived true}- :model/QueryAction _ {:action_id archived, :dataset_query (touch-category-query)}- :model/Action {in-collection :id} {:type :query, :name "In a collection", :collection_id collection-id}- :model/QueryAction _ {:action_id in-collection, :dataset_query (touch-category-query)}- :model/Action {on-model :id} {:type :query, :name "On a model", :model_id model-id}- :model/QueryAction _ {:action_id on-model, :dataset_query (touch-category-query)}]- (let [action-ids (fn [database-ids] (into #{} (map :id) (source/actions source/app-db-source database-ids)))- ours #{standalone archived in-collection on-model}]- (mt/with-current-user (mt/user->id :crowberto)- (testing "a database scope returns that database's unarchived actions without a model"- (is (= #{standalone in-collection} (set/intersection ours (action-ids #{(mt/id)})))))- (testing "no scope reads every database"- (is (= #{standalone in-collection} (set/intersection ours (action-ids nil)))))- (testing "a scope of another database leaves them out"- (is (= #{} (set/intersection ours (action-ids #{other-db-id})))))- (testing "an action renders with its parameters typed from the template tag"- (is (=? [{:kind "action"- :key "touchCategory"- :id standalone- :type "query"- :parameters [{:slug "id", :jsType "number", :required true}]}]- (filter #(= standalone (:id %)) (source/actions source/app-db-source #{(mt/id)})))))))))diff --git a/test/metabase/typed_schemas_rest/api/query_params_test.clj b/test/metabase/typed_schemas_rest/api/query_params_test.cljdeleted file mode 100644index c7ef14028b2..00000000000--- a/test/metabase/typed_schemas_rest/api/query_params_test.clj+++ /dev/null@@ -1,27 +0,0 @@-(ns metabase.typed-schemas-rest.api.query-params-test- (:require- [clojure.test :refer :all]- [metabase.typed-schemas-rest.api.query-params :as query-params]))--(deftest ^:parallel query-params->options-test- (is (= {:database {:name "Boba"}- :library-collection-refs [{:id 10} {:id 20}]- :include-data-library? false- :include-metric-library? false- :include-actions? true}- (query-params/query-params->options- {:database " Boba "- :library-collections " 10, 20 "- :include-actions true}))))--(deftest ^:parallel query-params->options-coerces-values-and-applies-defaults-test- (is (= {:database {:id 1}- :library-collection-refs [{:id 3}]- :include-data-library? true- :include-metric-library? true- :include-actions? false}- (query-params/query-params->options- {:database "1"- :library-collections "3"- :include-data-library "1"- :include-metric-library "true"}))))diff --git a/test/metabase/typed_schemas_rest/api_test.clj b/test/metabase/typed_schemas_rest/api_test.cljdeleted file mode 100644index 19886335ed2..00000000000--- a/test/metabase/typed_schemas_rest/api_test.clj+++ /dev/null@@ -1,129 +0,0 @@-(ns metabase.typed-schemas-rest.api-test- (:require- [clojure.string :as str]- [clojure.test :refer :all]- [metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata]- [metabase.permissions.core :as perms]- [metabase.test :as mt]- [metabase.test.fixtures :as fixtures]- [metabase.typed-schemas.core :as typed-schemas]- [toucan2.core :as t2]))--(use-fixtures :once (fixtures/initialize :db :web-server :test-users))--(deftest only-a-superuser-generates-the-schema-test- (testing "the schema is what an app's author builds from, and only an admin works with an app's repository"- (is (= "You don't have permissions to do that."- (mt/user-http-request :rasta :get 403 "typed-schemas/v1/typescript")))- (is (string? (mt/user-http-request :crowberto :get 200 "typed-schemas/v1/typescript")))))--(defn- without-generated-at- [typescript]- (str/replace typescript #"generatedAt: \"[^\"]+\"" "generatedAt: \"<generated-at>\""))--(deftest typescript-endpoint-test- (let [response (mt/user-http-request-full-response :crowberto :get 200 "typed-schemas/v1/typescript")]- (is (= "text/typescript; charset=utf-8" (get-in response [:headers "Content-Type"])))- ;; `includes?` rather than `starts-with?`: instances with compactable- ;; metrics render the pickFields helper before the first const.- (is (str/includes? (:body response) "\nconst schema = {"))- (is (str/includes? (:body response) "\n schemaVersion: 2"))- (is (not (str/includes? (:body response) "\n questions:")))- (is (str/includes? (:body response) "\n tables: tables"))- (is (str/includes? (:body response) "\n metrics: metrics"))- (is (str/ends-with? (:body response) "export default schema;\n"))- (is (not (str/includes? (:body response) "\"schemaVersion\"")))- (is (not (str/includes? (:body response) "operators: [ ]")))- (is (not (str/includes? (:body response) "parameters: [ ]")))- (is (not (str/includes? (:body response) "verified: false")))))--(deftest query-params-are-decoded-at-endpoint-test- (mt/with-dynamic-fn-redefs [typed-schemas/build-semantic-schema identity- typed-schemas/render-typescript pr-str]- (let [response (-> (mt/user-http-request-full-response- :crowberto- :get- 200- "typed-schemas/v1/typescript?include-actions=true&library-collections=1,2")- :body- read-string)]- (is (true? (:include-actions? response)))- (is (= [{:id 1} {:id 2}] (:library-collection-refs response))))))--(deftest database-filter-test- (let [database-id (mt/id)- database-name (t2/select-one-fn :name :model/Database :id database-id)- schema-by-id (:body (mt/user-http-request-full-response- :crowberto :get 200 "typed-schemas/v1/typescript" :database database-id))- schema-by-name (:body (mt/user-http-request-full-response- :crowberto :get 200 "typed-schemas/v1/typescript" :database database-name))- missing-schema (:body (mt/user-http-request-full-response- :crowberto :get 200 "typed-schemas/v1/typescript" :database "__missing_database__"))]- (testing "a database id and name select the same generated schema"- (is (= (without-generated-at schema-by-id)- (without-generated-at schema-by-name))))- (testing "a matching database includes its real tables"- (is (str/includes? schema-by-id " venues: {"))- (is (str/includes? schema-by-id "name: \"Venues\"")))- (testing "a non-matching database name returns an empty semantic schema"- (is (str/includes? missing-schema "schemaVersion: 2"))- (is (not (str/includes? missing-schema "const questions = { }")))- (is (str/includes? missing-schema "const tables = { }"))- (is (str/includes? missing-schema "const metrics = { }")))))--(deftest database-scope-includes-actions-only-when-requested-test- (mt/with-actions-enabled- (mt/with-temp [:model/Database other-db {}- :model/Card model {:name "Order model", :database_id (mt/id), :table_id (mt/id :orders)- :type :model- :dataset_query (let [mp (mt/metadata-provider)]- (lib/query mp (lib.metadata/table mp (mt/id :orders))))- :result_metadata [{:name "total", :display_name "Total"- :base_type :type/Float- :field_ref [:field (mt/id :orders :total) nil]- :id (mt/id :orders :total)}]}- :model/Action action {:name "Update order", :model_id (:id model), :type :implicit}- :model/ImplicitAction _ {:action_id (:id action), :kind "row/update"}- :model/Action standalone {:name "Discount order", :type :query}- :model/QueryAction _ {:action_id (:id standalone)- :dataset_query (lib/native-query (mt/metadata-provider)- "UPDATE orders SET discount = 0")}- :model/Collection copies {:name "Data App: orders", :namespace "data-apps"}- :model/Action copy {:name "Copied order", :type :query, :collection_id (:id copies)}- :model/QueryAction _ {:action_id (:id copy)- :dataset_query (lib/native-query (mt/metadata-provider)- "UPDATE orders SET discount = 0")}]- (let [schema (fn [& query-params]- (:body (apply mt/user-http-request-full-response- :crowberto :get 200 "typed-schemas/v1/typescript" query-params)))]- (testing "a database scope leaves actions out unless asked for"- (is (not (str/includes? (schema :database (mt/id)) "discountOrder"))))- (testing "include-actions adds the scoped database's actions without a model, and never a model"- (let [body (schema :database (mt/id) :include-actions true)]- (is (str/includes? body "discountOrder"))- (is (not (str/includes? body "updateOrder")))))- (testing "include-actions keeps actions of other databases out"- (is (not (str/includes? (schema :database (:id other-db) :include-actions true) "discountOrder"))))- (testing "include-models is refused"- (is (str/includes? (str (mt/user-http-request :crowberto :get 400 "typed-schemas/v1/typescript"- :include-models true))- "use include-actions")))- (testing "a copy a data app owns stays out"- (is (str/includes? (schema :database (mt/id) :include-actions true) "copiedOrder"))- (mt/with-dynamic-fn-redefs [perms/data-app-collection-ids (constantly #{(:id copies)})]- (is (not (str/includes? (schema :database (mt/id) :include-actions true) "copiedOrder")))))))))--(deftest collection-and-database-query-params-are-mutually-exclusive-test- (mt/user-http-request-full-response- :crowberto- :get- 400- "typed-schemas/v1/typescript?library-collections=1,2&database=1"))--(deftest question-collections-query-param-is-rejected-test- (mt/user-http-request-full-response- :crowberto- :get- 400- "typed-schemas/v1/typescript?question-collections=1"))