Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Generate data app files as an export writes them after a pull (#84406)

Alexander Polyankin29a4b7052ca3
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/app.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/app.clj
index 181d5c8dc74..8d72cb20072 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/generate/app.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/app.clj
@@ -7,7 +7,10 @@
[metabase-enterprise.data-apps.config :as data-app.config]
[metabase-enterprise.data-apps.schema :as data-apps.schema]
[metabase-enterprise.serialization.core :as serialization]
+ [metabase.collections.core :as collections]
+ [metabase.models.interface :as mi]
[metabase.util :as u]
+ [metabase.util.date-2 :as u.date]
[metabase.util.i18n :refer [tru]]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]))
@@ -16,7 +19,7 @@
(def ^:private bundle-path
"Where the app template builds its bundle, relative to the app's directory."
- "./dist/index.js")
+ "dist/index.js")
(defn- name->slug
"The slug `app-name` gives an app: its letters and numbers, lowercased, separated by single dashes."
@@ -33,16 +36,19 @@
(mu/defn generate :- ::data-apps.schema/app-files
"The `data_app.yaml` of a new app named `name`, served at `slug` (from the name by default), and its collection's
- file, both with new entity IDs."
+ file, both with new entity IDs and the YAML an export writes for them once a pull saves them."
[{app-name :name :keys [slug description]} :- ::data-apps.schema/app-request]
(let [slug (or slug (name->slug app-name))
_ (when-not (mr/validate ::data-apps.schema/slug slug)
(throw (ex-info (tru "Pass a slug: \"{0}\" is not a valid one." slug) {:status-code 400})))
app-id (u/generate-nano-id)
collection-id (u/generate-nano-id)
+ created-at (u.date/format (mi/current-timestamp))
collection {:name (str "Data App: " slug)
:namespace "data-apps"
:entity_id collection-id
+ :slug (collections/slugify (str "Data App: " slug))
+ :created_at created-at
:serdes/meta [{:model "Collection" :id collection-id
:label (serialization/slugify-name (str "Data App: " slug))}]}
app (m/assoc-some {:version data-app.config/supported-app-version
@@ -51,6 +57,7 @@
:path bundle-path
:collection collection-id
:entity_id app-id
+ :created_at created-at
:serdes/meta [{:model "DataApp" :id app-id :label slug}]}
:description description)]
{:files [(file app nil)
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/resources.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/resources.clj
index 519cb93e365..9b3b5e351b5 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/generate/resources.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/resources.clj
@@ -18,7 +18,9 @@
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.schema :as lib.schema]
+ [metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
+ [metabase.queries.core :as queries]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
@@ -139,12 +141,14 @@
query)))
(mu/defn- build-card :- :map
- "The unsaved question named `card-name` with entity ID `entity-id`, holding `query`."
+ "The question named `card-name` with entity ID `entity-id`, holding `query`, as a pull saves it."
[card-name :- ms/NonBlankString
entity-id :- ms/NanoIdString
query :- ::lib.schema/query]
{:entity_id entity-id
:name card-name
+ :created_at (mi/current-timestamp)
+ :card_schema queries/current-card-schema-version
:type :question
:display :table
:creator_id api/*current-user-id*
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj b/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj
index 6e2dfb1d3c2..3e26c998cd8 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj
@@ -172,7 +172,7 @@
(or (get files path)
(throw (ex-info (tru "Bundle file \"{0}\" not found." path)
{:status-code 400}))))))}}
- :defaults {:description nil :version 1 :allowed_hosts []}})
+ :defaults {:description nil :allowed_hosts []}})
(defmethod serdes/extract-query "DataApp"
[model-name {:keys [filter-column filter-ids] :as opts}]
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/app_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/app_test.clj
index 237a7547991..0a78d849ba7 100644
--- a/enterprise/backend/test/metabase_enterprise/data_apps/generate/app_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/app_test.clj
@@ -25,7 +25,7 @@
(testing "the manifest names the app, its bundle, its version and the collection it owns"
(is (=? {:name "Sales Ops"
:slug "sales-ops"
- :path "./dist/index.js"
+ :path "dist/index.js"
:version data-apps.config/supported-app-version
:collection collection-id
:entity_id string?
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj
index 15486971245..eeaaabe64d1 100644
--- a/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj
@@ -512,6 +512,8 @@
:type "question"
:display "table"
:creator_id "[email protected]"
+ :created_at string?
+ :card_schema pos-int?
:visualization_settings {:column_settings nil}
:parameters []
:parameter_mappings []
@@ -519,7 +521,7 @@
:stages [{:source-table (table-path "VENUES") :limit 5}]}}
entity))
(is (not-any? (partial contains? entity)
- [:description :collection_position :public_uuid :card_schema :archived :enable_embedding])
+ [:description :collection_position :public_uuid :archived :enable_embedding])
"what serialization leaves unset or at its default is left out, as the format omits it"))))
(deftest keys-come-in-the-order-serialization-writes-them-test
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.clj
index 99335222e26..79d6db89b5c 100644
--- a/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/serialization_test.clj
@@ -72,6 +72,7 @@
:slug "sales-ops"
:name "Sales Ops"
:description "Pipeline health"
+ :version 1
:path "dist/index.js"
:allowed_hosts ["https://api.example.com"]
:collection (t2/select-one-fn :entity_id :model/Collection :id (:resource_collection_id app))}
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj
index 41a684febd1..88f112a5c70 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj
@@ -10,7 +10,9 @@
[metabase-enterprise.remote-sync.impl :as impl]
[metabase-enterprise.remote-sync.models.remote-sync-object :as remote-sync.object]
[metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]
+ [metabase-enterprise.remote-sync.source :as source]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
+ [metabase-enterprise.remote-sync.spec :as spec]
[metabase-enterprise.remote-sync.test-helpers :as test-helpers]
[metabase.actions.core :as actions]
[metabase.collections.models.collection :as collection]
@@ -20,6 +22,7 @@
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
+ [metabase.util :as u]
[metabase.util.yaml :as yaml]
[toucan2.core :as t2]))
@@ -1040,3 +1043,41 @@
(is (= :success (:status result)) (:message result))))
(testing "the export is recorded: the rows are synced"
(is (empty? (remote-sync.object/dirty-rows)))))))))
+
+(defn- generated-app-tree
+ "The repo files of the `shop` app as `/api/apps/generate` writes them: its manifest and collection, a question
+ aggregating `metric-id`, the copy of that metric, the copy of `action-id`, and its bundle."
+ [metric-id action-id]
+ (let [{:keys [files]} (mt/user-http-request :crowberto :post 200 "apps/generate/app" {:name "Shop" :slug "shop"})
+ collection (first (filter #(str/starts-with? (:path %) "collections/") files))
+ collection-eid (:entity_id (yaml/parse-string (:yaml collection)))
+ folder (str (str/replace (:path collection) #"\.yaml$" "") "/")
+ resources (mt/user-http-request :crowberto :post 200 "apps/generate/resources"
+ {:queries [{:name "VenueCount"
+ :entity_id (u/generate-nano-id)
+ :collection_id collection-eid
+ :query {:stages [{:source {:type "table" :id (mt/id :venues)}
+ :aggregations [{:type "metric" :id metric-id}]}]}}]
+ :actions [{:action_id action-id
+ :entity_id (u/generate-nano-id)
+ :collection_id collection-eid}]})]
+ (merge (into {} (map (juxt :path :yaml)) files)
+ (into {} (map (fn [{:keys [file yaml]}] [(str folder file) yaml])) (mapcat resources [:queries :actions :metrics]))
+ {"data_apps/shop/dist/index.js" "BUNDLE"})))
+
+(deftest generated-files-are-what-an-export-writes-test
+ (testing "the files the generate endpoints write for an app are the ones an export writes once a pull saves them"
+ (with-data-apps-sync
+ (data-apps.tu/do-with-sources!
+ (fn [{:keys [metric-id action-id]}]
+ (t2/update! :model/Action action-id {:creator_id (mt/user->id :rasta)})
+ (let [tree (generated-app-tree metric-id action-id)
+ src (test-helpers/versioned-source :trees {"v0" tree} :current "v0")]
+ (is (= 6 (count (app-files-in tree))))
+ (let [result (import-at! src "v0" :force? true)]
+ (is (= :success (:status result)) (:message result)))
+ (let [exported (into {}
+ (map (juxt :path :content))
+ (source/serialize-specs (spec/extract-entities-for-export (spec/exportable-entities)) nil))]
+ (is (= (select-keys tree (app-files-in tree))
+ (select-keys exported (app-files-in exported)))))))))))
diff --git a/src/metabase/collections/core.clj b/src/metabase/collections/core.clj
index 9c0a8d199e5..8884cdb2967 100644
--- a/src/metabase/collections/core.clj
+++ b/src/metabase/collections/core.clj
@@ -44,6 +44,7 @@
remote-synced-collection?
remote-synced-dependents
shared-tenant-collection?
+ slugify
transforms-ns
data-apps-ns
data-actions-ns
diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.clj
index 14a94775c01..6bc25ac73ac 100644
--- a/src/metabase/collections/models/collection.clj
+++ b/src/metabase/collections/models/collection.clj
@@ -397,7 +397,9 @@
;;; | Slug Validation |
;;; +----------------------------------------------------------------------------------------------------------------+
-(defn- slugify [collection-name]
+(defn slugify
+ "The slug a collection named `collection-name` is stored with."
+ [collection-name]
;; double-check that someone isn't trying to use a blank string as the collection name
(when (str/blank? collection-name)
(throw (ex-info (tru "Collection name cannot be blank!")
diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.clj
index 61ea723a3b0..ddb68b6f6c4 100644
--- a/src/metabase/models/interface.clj
+++ b/src/metabase/models/interface.clj
@@ -46,6 +46,8 @@
[toucan2.util :as t2.u])
(:import
(java.sql Blob)
+ (java.time OffsetDateTime ZoneOffset)
+ (java.time.temporal ChronoUnit)
(toucan2.instance Instance)))
(set! *warn-on-reflection* true)
@@ -616,6 +618,11 @@
[]
(h2x/current-datetime-honeysql-form (mdb/db-type)))
+(defn current-timestamp
+ "The current moment in UTC at the microsecond precision of the app DB's timestamp columns, so it reads back unchanged."
+ []
+ (.truncatedTo (OffsetDateTime/now ZoneOffset/UTC) ChronoUnit/MICROS))
+
(defn- add-created-at-timestamp [obj & _]
(cond-> obj
(not (:created_at obj)) (assoc :created_at (now))))
diff --git a/src/metabase/queries/core.clj b/src/metabase/queries/core.clj
index 5374abe9bd1..807ca23f801 100644
--- a/src/metabase/queries/core.clj
+++ b/src/metabase/queries/core.clj
@@ -56,6 +56,7 @@
batch-fetch-dashboard-metadata
batch-fetch-query-metadata]
[metabase.queries.models.card
+ current-card-schema-version
fully-parameterized?
maybe-unverify!
model?
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index a4c571c0843..eb077297d1f 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -70,7 +70,7 @@
This value is used when loading old revision records that predate the schema versioning system."
20)
-(def ^:private current-schema-version
+(def current-card-schema-version
"Latest schema version number. This is an increasing integer stored in each card's `:card_schema` column.
It is used to guide `after-select` logic in how to modernize a card correctly.
@@ -84,13 +84,13 @@
version.
The core `after-select` logic compares each row's `card_schema` and runs the upgrade functions for all versions up to
- and including [[current-schema-version]]."
+ and including [[current-card-schema-version]]."
24)
(defmulti ^:private upgrade-card-schema-to
"Upgrades a card on read, so that it fits the given schema version number.
- The number is the **target** revision - if we read a row with `:card_schema` 22 and the [[current-schema-version]]
+ The number is the **target** revision - if we read a row with `:card_schema` 22 and the [[current-card-schema-version]]
is 25, then the `after-select` logic will effectively call
(-> card
@@ -461,7 +461,7 @@
(let [card (lib/normalize ::queries.schema/card card)
defaults {:parameters []
:parameter_mappings []
- :card_schema current-schema-version}
+ :card_schema current-card-schema-version}
card (maybe-check-dashboard-internal-card
(merge defaults card))]
(u/prog1 card
@@ -760,7 +760,7 @@
:else
(loop [card card]
;; Use >= to allow for downgrades.
- (if (>= (:card_schema card) current-schema-version)
+ (if (>= (:card_schema card) current-card-schema-version)
card
(let [new-version (inc (:card_schema card))]
(recur (assoc (upgrade-card-schema-to card new-version)
@@ -782,7 +782,7 @@
(let [stored-schema (:card_schema original)]
(if-not (and stored-schema
;; Use < to leave downgraded rows alone, matching [[upgrade-card-schema-to-latest]].
- (< stored-schema current-schema-version)
+ (< stored-schema current-card-schema-version)
(some #(contains? changes %) card-schema/schema-upgrade-triggers))
card
;; Force the `update-baseline-select?` flag to false, since this runs inside the `t2/before-update` process.
@@ -799,7 +799,7 @@
relevant (when (seq columns)
(select-keys upgraded columns))]
(-> (merge card relevant)
- (assoc :card_schema current-schema-version))))))
+ (assoc :card_schema current-card-schema-version))))))
(defonce ^:private unique-cards-with-blank-dataset-query
(atom #{}))
@@ -850,7 +850,7 @@
;; +===============================================================================================+
;; | DO NOT EDIT THIS FUNCTION DIRECTLY! |
;; | Future revisions to the shapes of cards should be handled via [[upgrade-card-schema-to]]. |
- ;; | See [[current-schema-version]] for details on the schema versioning. |
+ ;; | See [[current-card-schema-version]] for details on the schema versioning. |
;; +===============================================================================================+
(-> card
(dissoc :dataset_query_metrics_v2_migration_backup)
@@ -1047,7 +1047,7 @@
(u/prog1
(-> card
(assoc :metabase_version config/mb-version-string
- :card_schema current-schema-version)
+ :card_schema current-card-schema-version)
normalize-card
;; Must have an entity_id before populating the metadata. TODO (Cam 7/11/25) -- actually, this is no longer true,
;; since we're removing `:ident`s; we can probably remove this now.
diff --git a/test/metabase/metrics/api_dimension_test.clj b/test/metabase/metrics/api_dimension_test.clj
index c2f40d18161..b0328afde08 100644
--- a/test/metabase/metrics/api_dimension_test.clj
+++ b/test/metabase/metrics/api_dimension_test.clj
@@ -815,7 +815,7 @@
(t2/update! :model/Card metric-id {:dataset_query (people-metric-query)})
(let [{:keys [card_schema dimensions]} (stored-card metric-id)
stored-names (display-name-by-field-id dimensions)]
- (is (= @#'card/current-schema-version card_schema)
+ (is (= card/current-card-schema-version card_schema)
"the vintage advances alongside the migrated columns")
(testing "the dimensions written alongside that bump are the modernized ones"
(is (= "Product - ID" (stored-names (mt/id :products :id))))
@@ -832,7 +832,7 @@
(t2/update! :model/Card metric-id {:dimensions modernized})
(let [{:keys [card_schema dimensions]} (stored-card metric-id)
stored-names (display-name-by-field-id dimensions)]
- (is (= @#'card/current-schema-version card_schema))
+ (is (= card/current-card-schema-version card_schema))
(is (= "Product - ID" (stored-names (mt/id :products :id))))
(is (= "User - ID" (stored-names (mt/id :people :id))))
(doseq [[field-id display-name] stored-names]
diff --git a/test/metabase/models/interface_test.clj b/test/metabase/models/interface_test.clj
index baeefb20158..fd5ceb66564 100644
--- a/test/metabase/models/interface_test.clj
+++ b/test/metabase/models/interface_test.clj
@@ -7,6 +7,7 @@
[metabase.models.interface :as mi]
[metabase.test :as mt]
[metabase.util :as u]
+ [metabase.util.date-2 :as u.date]
[metabase.util.encryption :as encryption]
[metabase.util.encryption-test :as encryption-test]
[metabase.util.json :as json]
@@ -122,6 +123,13 @@
:updated_at expected-timestamp}
field)))))))
+(deftest ^:parallel current-timestamp-reads-back-unchanged-test
+ (testing "a timestamp from current-timestamp reads back from the app DB as the same text an export writes"
+ (let [now (mi/current-timestamp)]
+ (mt/with-temp [:model/Field field {:created_at now}]
+ (is (= (u.date/format now)
+ (u.date/format (t2/select-one-fn :created_at :model/Field (:id field)))))))))
+
(defmethod mi/non-timestamped-fields :test-model/updated-at-tester [_]
#{:non_timestamped :other})
diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.clj
index 9df7f7df9a9..b646eba4f50 100644
--- a/test/metabase/queries/models/card_test.clj
+++ b/test/metabase/queries/models/card_test.clj
@@ -1019,7 +1019,7 @@
(deftest ^:parallel upgrade-card-schema-after-downgrade
(testing "We exit the loop if a card_schema is higher than the current schema."
- (let [card (assoc complete-card :card_schema (inc @#'card/current-schema-version))]
+ (let [card (assoc complete-card :card_schema (inc card/current-card-schema-version))]
(is (= card
(#'card/upgrade-card-schema-to-latest card))))))
@@ -1030,7 +1030,7 @@
"every instance rather than only the ones holding an out-of-date card.")
(doseq [omitted (disj card-schema/schema-upgrade-triggers :card_schema)]
(testing (str "omitting " omitted)
- (let [card (dissoc (assoc complete-card :card_schema @#'card/current-schema-version) omitted)]
+ (let [card (dissoc (assoc complete-card :card_schema card/current-card-schema-version) omitted)]
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Cannot SELECT a Card with card_schema columns without all columns needed for an upgrade"
@@ -1452,7 +1452,7 @@
(mt/with-temp [:model/Card {card-id :id} {}]
(age-card! card-id 20)
(t2/update! :model/Card card-id {:dataset_query (mt/mbql-query venues)})
- (is (= @#'card/current-schema-version (stored-card-schema card-id))
+ (is (= card/current-card-schema-version (stored-card-schema card-id))
(str "the bump must reach the database. t2/update! diffs changes against a baseline SELECT; if "
"the after-select schema upgrade runs on that baseline it already reads as current, the "
"diff drops :card_schema, and the row stays behind forever."))))))
@@ -2276,6 +2276,6 @@
(testing ":dataset_query is still normalized to current MBQL"
(is (=? {:lib/type :mbql/query} (:dataset_query card))))
(testing "and the row still reports the current schema version"
- (is (= @#'card/current-schema-version (:card_schema card))))))
+ (is (= card/current-card-schema-version (:card_schema card))))))
(testing "the stored row is untouched by any of this"
(is (= 23 (stored-card-schema card-id)))))))