The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Generate data app files as an export writes them after a pull (#84406)
Alexander Polyankin29a4b7052ca3
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/app.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/app.cljindex 181d5c8dc74..8d72cb20072 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/generate/app.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/app.clj@@ -7,7 +7,10 @@[metabase-enterprise.data-apps.config :as data-app.config][metabase-enterprise.data-apps.schema :as data-apps.schema][metabase-enterprise.serialization.core :as serialization]+ [metabase.collections.core :as collections]+ [metabase.models.interface :as mi][metabase.util :as u]+ [metabase.util.date-2 :as u.date][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]))@@ -16,7 +19,7 @@(def ^:private bundle-path"Where the app template builds its bundle, relative to the app's directory."- "./dist/index.js")+ "dist/index.js")(defn- name->slug"The slug `app-name` gives an app: its letters and numbers, lowercased, separated by single dashes."@@ -33,16 +36,19 @@(mu/defn generate :- ::data-apps.schema/app-files"The `data_app.yaml` of a new app named `name`, served at `slug` (from the name by default), and its collection's- file, both with new entity IDs."+ file, both with new entity IDs and the YAML an export writes for them once a pull saves them."[{app-name :name :keys [slug description]} :- ::data-apps.schema/app-request](let [slug (or slug (name->slug app-name))_ (when-not (mr/validate ::data-apps.schema/slug slug)(throw (ex-info (tru "Pass a slug: \"{0}\" is not a valid one." slug) {:status-code 400})))app-id (u/generate-nano-id)collection-id (u/generate-nano-id)+ created-at (u.date/format (mi/current-timestamp))collection {:name (str "Data App: " slug):namespace "data-apps":entity_id collection-id+ :slug (collections/slugify (str "Data App: " slug))+ :created_at created-at:serdes/meta [{:model "Collection" :id collection-id:label (serialization/slugify-name (str "Data App: " slug))}]}app (m/assoc-some {:version data-app.config/supported-app-version@@ -51,6 +57,7 @@:path bundle-path:collection collection-id:entity_id app-id+ :created_at created-at:serdes/meta [{:model "DataApp" :id app-id :label slug}]}:description description)]{:files [(file app nil)diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/resources.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/resources.cljindex 519cb93e365..9b3b5e351b5 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/generate/resources.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/resources.clj@@ -18,7 +18,9 @@[metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.schema :as lib.schema]+ [metabase.models.interface :as mi][metabase.models.serialization :as serdes]+ [metabase.queries.core :as queries][metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]@@ -139,12 +141,14 @@query)))(mu/defn- build-card :- :map- "The unsaved question named `card-name` with entity ID `entity-id`, holding `query`."+ "The question named `card-name` with entity ID `entity-id`, holding `query`, as a pull saves it."[card-name :- ms/NonBlankStringentity-id :- ms/NanoIdStringquery :- ::lib.schema/query]{:entity_id entity-id:name card-name+ :created_at (mi/current-timestamp)+ :card_schema queries/current-card-schema-version:type :question:display :table:creator_id api/*current-user-id*diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj b/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.cljindex 6e2dfb1d3c2..3e26c998cd8 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj@@ -172,7 +172,7 @@(or (get files path)(throw (ex-info (tru "Bundle file \"{0}\" not found." path){:status-code 400}))))))}}- :defaults {:description nil :version 1 :allowed_hosts []}})+ :defaults {:description nil :allowed_hosts []}})(defmethod serdes/extract-query "DataApp"[model-name {:keys [filter-column filter-ids] :as opts}]diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/app_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/app_test.cljindex 237a7547991..0a78d849ba7 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/generate/app_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/app_test.clj@@ -25,7 +25,7 @@(testing "the manifest names the app, its bundle, its version and the collection it owns"(is (=? {:name "Sales Ops":slug "sales-ops"- :path "./dist/index.js"+ :path "dist/index.js":version data-apps.config/supported-app-version:collection collection-id:entity_id string?diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.cljindex 15486971245..eeaaabe64d1 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj@@ -512,6 +512,8 @@:type "question":display "table":creator_id "[email protected]"+ :created_at string?+ :card_schema pos-int?:visualization_settings {:column_settings nil}:parameters []:parameter_mappings []@@ -519,7 +521,7 @@:stages [{:source-table (table-path "VENUES") :limit 5}]}}entity))(is (not-any? (partial contains? entity)- [:description :collection_position :public_uuid :card_schema :archived :enable_embedding])+ [:description :collection_position :public_uuid :archived :enable_embedding])"what serialization leaves unset or at its default is left out, as the format omits it"))))(deftest keys-come-in-the-order-serialization-writes-them-testdiff --git a/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.cljindex 99335222e26..79d6db89b5c 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.clj@@ -72,6 +72,7 @@:slug "sales-ops":name "Sales Ops":description "Pipeline health"+ :version 1:path "dist/index.js":allowed_hosts ["https://api.example.com"]:collection (t2/select-one-fn :entity_id :model/Collection :id (:resource_collection_id app))}diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.cljindex 41a684febd1..88f112a5c70 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj@@ -10,7 +10,9 @@[metabase-enterprise.remote-sync.impl :as impl][metabase-enterprise.remote-sync.models.remote-sync-object :as remote-sync.object][metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]+ [metabase-enterprise.remote-sync.source :as source][metabase-enterprise.remote-sync.source.protocol :as source.p]+ [metabase-enterprise.remote-sync.spec :as spec][metabase-enterprise.remote-sync.test-helpers :as test-helpers][metabase.actions.core :as actions][metabase.collections.models.collection :as collection]@@ -20,6 +22,7 @@[metabase.search.test-util :as search.tu][metabase.test :as mt][metabase.test.fixtures :as fixtures]+ [metabase.util :as u][metabase.util.yaml :as yaml][toucan2.core :as t2]))@@ -1040,3 +1043,41 @@(is (= :success (:status result)) (:message result))))(testing "the export is recorded: the rows are synced"(is (empty? (remote-sync.object/dirty-rows)))))))))++(defn- generated-app-tree+ "The repo files of the `shop` app as `/api/apps/generate` writes them: its manifest and collection, a question+ aggregating `metric-id`, the copy of that metric, the copy of `action-id`, and its bundle."+ [metric-id action-id]+ (let [{:keys [files]} (mt/user-http-request :crowberto :post 200 "apps/generate/app" {:name "Shop" :slug "shop"})+ collection (first (filter #(str/starts-with? (:path %) "collections/") files))+ collection-eid (:entity_id (yaml/parse-string (:yaml collection)))+ folder (str (str/replace (:path collection) #"\.yaml$" "") "/")+ resources (mt/user-http-request :crowberto :post 200 "apps/generate/resources"+ {:queries [{:name "VenueCount"+ :entity_id (u/generate-nano-id)+ :collection_id collection-eid+ :query {:stages [{:source {:type "table" :id (mt/id :venues)}+ :aggregations [{:type "metric" :id metric-id}]}]}}]+ :actions [{:action_id action-id+ :entity_id (u/generate-nano-id)+ :collection_id collection-eid}]})]+ (merge (into {} (map (juxt :path :yaml)) files)+ (into {} (map (fn [{:keys [file yaml]}] [(str folder file) yaml])) (mapcat resources [:queries :actions :metrics]))+ {"data_apps/shop/dist/index.js" "BUNDLE"})))++(deftest generated-files-are-what-an-export-writes-test+ (testing "the files the generate endpoints write for an app are the ones an export writes once a pull saves them"+ (with-data-apps-sync+ (data-apps.tu/do-with-sources!+ (fn [{:keys [metric-id action-id]}]+ (t2/update! :model/Action action-id {:creator_id (mt/user->id :rasta)})+ (let [tree (generated-app-tree metric-id action-id)+ src (test-helpers/versioned-source :trees {"v0" tree} :current "v0")]+ (is (= 6 (count (app-files-in tree))))+ (let [result (import-at! src "v0" :force? true)]+ (is (= :success (:status result)) (:message result)))+ (let [exported (into {}+ (map (juxt :path :content))+ (source/serialize-specs (spec/extract-entities-for-export (spec/exportable-entities)) nil))]+ (is (= (select-keys tree (app-files-in tree))+ (select-keys exported (app-files-in exported)))))))))))diff --git a/src/metabase/collections/core.clj b/src/metabase/collections/core.cljindex 9c0a8d199e5..8884cdb2967 100644--- a/src/metabase/collections/core.clj+++ b/src/metabase/collections/core.clj@@ -44,6 +44,7 @@remote-synced-collection?remote-synced-dependentsshared-tenant-collection?+ slugifytransforms-nsdata-apps-nsdata-actions-nsdiff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.cljindex 14a94775c01..6bc25ac73ac 100644--- a/src/metabase/collections/models/collection.clj+++ b/src/metabase/collections/models/collection.clj@@ -397,7 +397,9 @@;;; | Slug Validation |;;; +----------------------------------------------------------------------------------------------------------------+-(defn- slugify [collection-name]+(defn slugify+ "The slug a collection named `collection-name` is stored with."+ [collection-name];; double-check that someone isn't trying to use a blank string as the collection name(when (str/blank? collection-name)(throw (ex-info (tru "Collection name cannot be blank!")diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.cljindex 61ea723a3b0..ddb68b6f6c4 100644--- a/src/metabase/models/interface.clj+++ b/src/metabase/models/interface.clj@@ -46,6 +46,8 @@[toucan2.util :as t2.u])(:import(java.sql Blob)+ (java.time OffsetDateTime ZoneOffset)+ (java.time.temporal ChronoUnit)(toucan2.instance Instance)))(set! *warn-on-reflection* true)@@ -616,6 +618,11 @@[](h2x/current-datetime-honeysql-form (mdb/db-type)))+(defn current-timestamp+ "The current moment in UTC at the microsecond precision of the app DB's timestamp columns, so it reads back unchanged."+ []+ (.truncatedTo (OffsetDateTime/now ZoneOffset/UTC) ChronoUnit/MICROS))+(defn- add-created-at-timestamp [obj & _](cond-> obj(not (:created_at obj)) (assoc :created_at (now))))diff --git a/src/metabase/queries/core.clj b/src/metabase/queries/core.cljindex 5374abe9bd1..807ca23f801 100644--- a/src/metabase/queries/core.clj+++ b/src/metabase/queries/core.clj@@ -56,6 +56,7 @@batch-fetch-dashboard-metadatabatch-fetch-query-metadata][metabase.queries.models.card+ current-card-schema-versionfully-parameterized?maybe-unverify!model?diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex a4c571c0843..eb077297d1f 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -70,7 +70,7 @@This value is used when loading old revision records that predate the schema versioning system."20)-(def ^:private current-schema-version+(def current-card-schema-version"Latest schema version number. This is an increasing integer stored in each card's `:card_schema` column.It is used to guide `after-select` logic in how to modernize a card correctly.@@ -84,13 +84,13 @@version.The core `after-select` logic compares each row's `card_schema` and runs the upgrade functions for all versions up to- and including [[current-schema-version]]."+ and including [[current-card-schema-version]]."24)(defmulti ^:private upgrade-card-schema-to"Upgrades a card on read, so that it fits the given schema version number.- The number is the **target** revision - if we read a row with `:card_schema` 22 and the [[current-schema-version]]+ The number is the **target** revision - if we read a row with `:card_schema` 22 and the [[current-card-schema-version]]is 25, then the `after-select` logic will effectively call(-> card@@ -461,7 +461,7 @@(let [card (lib/normalize ::queries.schema/card card)defaults {:parameters []:parameter_mappings []- :card_schema current-schema-version}+ :card_schema current-card-schema-version}card (maybe-check-dashboard-internal-card(merge defaults card))](u/prog1 card@@ -760,7 +760,7 @@:else(loop [card card];; Use >= to allow for downgrades.- (if (>= (:card_schema card) current-schema-version)+ (if (>= (:card_schema card) current-card-schema-version)card(let [new-version (inc (:card_schema card))](recur (assoc (upgrade-card-schema-to card new-version)@@ -782,7 +782,7 @@(let [stored-schema (:card_schema original)](if-not (and stored-schema;; Use < to leave downgraded rows alone, matching [[upgrade-card-schema-to-latest]].- (< stored-schema current-schema-version)+ (< stored-schema current-card-schema-version)(some #(contains? changes %) card-schema/schema-upgrade-triggers))card;; Force the `update-baseline-select?` flag to false, since this runs inside the `t2/before-update` process.@@ -799,7 +799,7 @@relevant (when (seq columns)(select-keys upgraded columns))](-> (merge card relevant)- (assoc :card_schema current-schema-version))))))+ (assoc :card_schema current-card-schema-version))))))(defonce ^:private unique-cards-with-blank-dataset-query(atom #{}))@@ -850,7 +850,7 @@;; +===============================================================================================+;; | DO NOT EDIT THIS FUNCTION DIRECTLY! |;; | Future revisions to the shapes of cards should be handled via [[upgrade-card-schema-to]]. |- ;; | See [[current-schema-version]] for details on the schema versioning. |+ ;; | See [[current-card-schema-version]] for details on the schema versioning. |;; +===============================================================================================+(-> card(dissoc :dataset_query_metrics_v2_migration_backup)@@ -1047,7 +1047,7 @@(u/prog1(-> card(assoc :metabase_version config/mb-version-string- :card_schema current-schema-version)+ :card_schema current-card-schema-version)normalize-card;; Must have an entity_id before populating the metadata. TODO (Cam 7/11/25) -- actually, this is no longer true,;; since we're removing `:ident`s; we can probably remove this now.diff --git a/test/metabase/metrics/api_dimension_test.clj b/test/metabase/metrics/api_dimension_test.cljindex c2f40d18161..b0328afde08 100644--- a/test/metabase/metrics/api_dimension_test.clj+++ b/test/metabase/metrics/api_dimension_test.clj@@ -815,7 +815,7 @@(t2/update! :model/Card metric-id {:dataset_query (people-metric-query)})(let [{:keys [card_schema dimensions]} (stored-card metric-id)stored-names (display-name-by-field-id dimensions)]- (is (= @#'card/current-schema-version card_schema)+ (is (= card/current-card-schema-version card_schema)"the vintage advances alongside the migrated columns")(testing "the dimensions written alongside that bump are the modernized ones"(is (= "Product - ID" (stored-names (mt/id :products :id))))@@ -832,7 +832,7 @@(t2/update! :model/Card metric-id {:dimensions modernized})(let [{:keys [card_schema dimensions]} (stored-card metric-id)stored-names (display-name-by-field-id dimensions)]- (is (= @#'card/current-schema-version card_schema))+ (is (= card/current-card-schema-version card_schema))(is (= "Product - ID" (stored-names (mt/id :products :id))))(is (= "User - ID" (stored-names (mt/id :people :id))))(doseq [[field-id display-name] stored-names]diff --git a/test/metabase/models/interface_test.clj b/test/metabase/models/interface_test.cljindex baeefb20158..fd5ceb66564 100644--- a/test/metabase/models/interface_test.clj+++ b/test/metabase/models/interface_test.clj@@ -7,6 +7,7 @@[metabase.models.interface :as mi][metabase.test :as mt][metabase.util :as u]+ [metabase.util.date-2 :as u.date][metabase.util.encryption :as encryption][metabase.util.encryption-test :as encryption-test][metabase.util.json :as json]@@ -122,6 +123,13 @@:updated_at expected-timestamp}field)))))))+(deftest ^:parallel current-timestamp-reads-back-unchanged-test+ (testing "a timestamp from current-timestamp reads back from the app DB as the same text an export writes"+ (let [now (mi/current-timestamp)]+ (mt/with-temp [:model/Field field {:created_at now}]+ (is (= (u.date/format now)+ (u.date/format (t2/select-one-fn :created_at :model/Field (:id field)))))))))+(defmethod mi/non-timestamped-fields :test-model/updated-at-tester [_]#{:non_timestamped :other})diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.cljindex 9df7f7df9a9..b646eba4f50 100644--- a/test/metabase/queries/models/card_test.clj+++ b/test/metabase/queries/models/card_test.clj@@ -1019,7 +1019,7 @@(deftest ^:parallel upgrade-card-schema-after-downgrade(testing "We exit the loop if a card_schema is higher than the current schema."- (let [card (assoc complete-card :card_schema (inc @#'card/current-schema-version))]+ (let [card (assoc complete-card :card_schema (inc card/current-card-schema-version))](is (= card(#'card/upgrade-card-schema-to-latest card))))))@@ -1030,7 +1030,7 @@"every instance rather than only the ones holding an out-of-date card.")(doseq [omitted (disj card-schema/schema-upgrade-triggers :card_schema)](testing (str "omitting " omitted)- (let [card (dissoc (assoc complete-card :card_schema @#'card/current-schema-version) omitted)]+ (let [card (dissoc (assoc complete-card :card_schema card/current-card-schema-version) omitted)](is (thrown-with-msg?clojure.lang.ExceptionInfo#"Cannot SELECT a Card with card_schema columns without all columns needed for an upgrade"@@ -1452,7 +1452,7 @@(mt/with-temp [:model/Card {card-id :id} {}](age-card! card-id 20)(t2/update! :model/Card card-id {:dataset_query (mt/mbql-query venues)})- (is (= @#'card/current-schema-version (stored-card-schema card-id))+ (is (= card/current-card-schema-version (stored-card-schema card-id))(str "the bump must reach the database. t2/update! diffs changes against a baseline SELECT; if ""the after-select schema upgrade runs on that baseline it already reads as current, the ""diff drops :card_schema, and the row stays behind forever."))))))@@ -2276,6 +2276,6 @@(testing ":dataset_query is still normalized to current MBQL"(is (=? {:lib/type :mbql/query} (:dataset_query card))))(testing "and the row still reports the current schema version"- (is (= @#'card/current-schema-version (:card_schema card))))))+ (is (= card/current-card-schema-version (:card_schema card))))))(testing "the stored row is untouched by any of this"(is (= 23 (stored-card-schema card-id)))))))