Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Rehab the `:card_schema` read-time upgrade (#82186)

Braden Shepherdsonb4a88f04b4ac
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 6f3c40dcf34..085f47cf760 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2044,7 +2044,8 @@
queries
{:team "Graphy"
- :api #{metabase.queries.core
+ :api #{metabase.queries.card-schema
+ metabase.queries.core
metabase.queries.init
metabase.queries.models.query ; TODO FIXME
metabase.queries.schema}
@@ -2990,6 +2991,7 @@
metrics
models
permissions
+ queries
system
util
warehouse-schema-overlay}
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index 11e96cc5abf..a1f7ca90a96 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -22,7 +22,7 @@
:equals-true 1
:main-without-gen-class 1
:metabase/disallow-hardcoded-driver-names-in-tests 34
- :metabase/discourage-dynamic-vars 395
+ :metabase/discourage-dynamic-vars 396
:metabase/discourage-millis-duration 1
:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests 13
:metabase/modules 24
diff --git a/e2e/test/scenarios/collections/cleanup.cy.spec.js b/e2e/test/scenarios/collections/cleanup.cy.spec.js
index 17a5813903c..1a334f82027 100644
--- a/e2e/test/scenarios/collections/cleanup.cy.spec.js
+++ b/e2e/test/scenarios/collections/cleanup.cy.spec.js
@@ -240,7 +240,9 @@ describe("scenarios > collections > clean up", () => {
isMatching(
{
event: "moved-to-trash",
- event_detail: P.union("dashboard", "question"),
+ // "dataset" because `bulkCreateQuestions` seeds `type: "model"` cards, and the
+ // stale listing labels those `model: "dataset"`.
+ event_detail: P.union("dashboard", "question", "dataset"),
target_id: P.number,
triggered_from: "cleanup_modal",
duration_ms: P.number,
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
index a5482257dc6..a707de73f0c 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/complexity.clj
@@ -232,8 +232,7 @@
;; Extra columns (`:collection_id`, `:is_published`, `:visibility_type`, `:db_id`,
;; `:data_layer`, `:data_authority`) are selected purely to drive the in-memory
;; library/metabot derivations below — they're ignored by `->card-entity` /
- ;; `->table-entity`. `:card_schema` is required by `:model/Card`'s post-select hooks
- ;; even when we don't otherwise use it.
+ ;; `->table-entity`.
universe-cards (data-complexity-score.db/universe-cards audit/audit-db-id)
universe-tables (data-complexity-score.db/universe-tables audit/audit-db-id)
field-counts (table-field-counts (mapv :id universe-tables))
diff --git a/enterprise/backend/src/metabase_enterprise/data_complexity_score/db.clj b/enterprise/backend/src/metabase_enterprise/data_complexity_score/db.clj
index 74d2a67560c..c5eae3c0024 100644
--- a/enterprise/backend/src/metabase_enterprise/data_complexity_score/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_complexity_score/db.clj
@@ -55,7 +55,7 @@
"The ID, name, type, and Collection of the unarchived metric and model Cards outside the Database with
`audit-database-id`."
[audit-database-id :- ::lib.schema.id/database]
- (t2/select [:model/Card :id :name :type :collection_id :card_schema]
+ (t2/select [:model/Card :id :name :type :collection_id]
:type [:in ["metric" "model"]]
:archived false
:database_id [:not= audit-database-id]))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
index 19ca533dfe0..6ff73ed6fa4 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
@@ -204,8 +204,10 @@
(def ^:private entity-select-fields
{:card [:id :name :description :type :display :database_id :query_type :collection_id :dashboard_id :document_id :result_metadata
:created_at :creator_id :view_count
- ;; :card_schema always has to be selected
- :card_schema]
+ ;; :card_schema always has to be selected, and every column the Card schema upgrade
+ ;; reads has to come with it. The last three are not part of the response — `entity-keys`
+ ;; decides that — they are here only to make the upgrade runnable.
+ :card_schema :dataset_query :dimensions :dimension_mappings]
:dashboard [:id :name :description :created_at :creator_id :collection_id :view_count]
:document [:id :name :created_at :creator_id :collection_id :view_count]
:table [:id :name :description :display_name :db_id :schema
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
index 8652b590536..bc26addcb97 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
@@ -581,12 +581,12 @@
(mu/defn card-types-by-id
"A map of Card ID to type for `card-ids`."
[card-ids :- [:sequential ::lib.schema.id/card]]
- (t2/select-fn->fn :id :type [:model/Card :id :type :card_schema] :id [:in card-ids]))
+ (t2/select-fn->fn :id :type [:model/Card :id :type] :id [:in card-ids]))
(mu/defn card-database-ids
- "The `:id`, `:database_id`, and `:card_schema` of the Cards with `card-ids`."
+ "The `:id` and `:database_id` of the Cards with `card-ids`."
[card-ids :- [:sequential ::lib.schema.id/card]]
- (t2/select [:model/Card :id :database_id :card_schema] :id [:in card-ids]))
+ (t2/select [:model/Card :id :database_id] :id [:in card-ids]))
(mu/defn set-card-result-metadata!
"Set the result metadata of the Card with `card-id`, returning the number updated."
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/db.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/db.clj
index 1c8dc2de4d3..b27b3cda3bb 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/db.clj
@@ -13,7 +13,7 @@
`collection-ids`, optionally narrowed to just `id`."
[collection-ids :- [:sequential ::lib.schema.id/collection]
id :- [:maybe ::lib.schema.id/card]]
- (t2/select [:model/Card :id :name :description :type :card_schema]
+ (t2/select [:model/Card :id :name :description :type]
{:where [:and
[:in :collection_id collection-ids]
[:= :archived false]
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj
index 39d23accaf5..a5f138e2bda 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj
@@ -157,8 +157,7 @@
(vec (distinct (cons (:id lib) (collections/descendant-ids lib)))))
(defn- library-cards [lib-ids id]
- ;; :card_schema is mandatory in any column-scoped Card SELECT (toucan guard). Card :type is keywordized
- ;; (:metric / :model); the entity_type is its name string.
+ ;; Card :type is keywordized (:metric / :model); the entity_type is its name string.
(->> (entity-retrieval.db/library-cards-in-collections lib-ids id)
(map (fn [c] (->library-entity (name (:type c)) (:id c) (:name c) (:description c))))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj
index 12eca1e727e..e36518ea0a4 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/core.clj
@@ -195,7 +195,6 @@
"card")
;; Named for the wire, as the dirty-changes payload already does for `display`.
:display (some-> display name)}]))
- ;; :card_schema is required alongside :type — selecting it runs Card's schema upgrades.
(remote-sync.db/card-types ids))))
(defn- describe-entities
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
index 5322d7ca3dc..d03a3a98ce7 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
@@ -288,9 +288,9 @@
:where (path-expr paths true)}))
(mu/defn card-types
- "The `:id`, `:type`, :display, and `:card_schema` of the Cards with `card-ids`."
+ "The `:id`, `:type`, and `:display` of the Cards with `card-ids`."
[card-ids :- [:sequential ::lib.schema.id/card]]
- (t2/select [:model/Card :id :type :display :card_schema] :id [:in card-ids]))
+ (t2/select [:model/Card :id :type :display] :id [:in card-ids]))
(mu/defn user-settings-exist-for-table?
"Whether the Table with `table-id`, or any of its Fields, has a user-settings row."
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/db.clj b/enterprise/backend/src/metabase_enterprise/sandbox/db.clj
index a4e1e9f7587..03b8b1527bd 100644
--- a/enterprise/backend/src/metabase_enterprise/sandbox/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/sandbox/db.clj
@@ -6,8 +6,10 @@
[metabase-enterprise.sandbox.schema :as sandbox.schema]
[metabase.app-db.core :as mdb]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.queries.core :as queries]
[metabase.queries.schema :as queries.schema]
[metabase.users.schema :as users.schema]
+ [metabase.util :as u]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
@@ -181,14 +183,14 @@
(t2/select [:model/Field :id :name] :table_id table-id :name [:in field-names] {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn cards-by-id
- "A map of Card ID to the query, result metadata, and schema of the Cards with `card-ids`."
+ "A map of Card ID to the query-related columns of the Cards with `card-ids`."
[card-ids :- [:set ::lib.schema.id/card]]
- (t2/select-pk->fn identity [:model/Card :id :dataset_query :result_metadata :card_schema] :id [:in card-ids]))
+ (u/index-by :id (queries/cards-queries-info card-ids)))
(mu/defn cards-result-metadata
- "The `:id`, `:result_metadata`, and `:card_schema` of the Cards with `card-ids`."
+ "The query-related columns of the Cards with `card-ids`."
[card-ids :- [:set ::lib.schema.id/card]]
- (t2/select [:model/Card :id :result_metadata :card_schema] :id [:in card-ids]))
+ (queries/cards-queries-info card-ids))
(mu/defn card-result-metadata
"The result metadata of the Card with `card-id`."
@@ -196,10 +198,12 @@
(t2/select-one-fn :result_metadata :model/Card :id card-id))
(mu/defn sandboxing-cards
- "The `:id`, `:dataset_query`, `:database_id`, and `:card_schema` of the Cards Sandboxes are built on."
+ "The `:id`, `:dataset_query`, and `:database_id` of the Cards Sandboxes are built on."
[]
(t2/select :model/Card
- {:select [:c.id :c.dataset_query :c.database_id :c.card_schema]
+ {:select [:c.id :c.dataset_query :c.database_id :c.card_schema
+ ;; required alongside :card_schema for the Card schema upgrade
+ :c.type :c.result_metadata :c.dimensions :c.dimension_mappings]
:from [[(t2/table-name :model/Card) :c]]
:where [:exists ^:allow-subquery {:select [[[:inline 1]]]
:from [[(t2/table-name :model/Sandbox) :s]]
diff --git a/enterprise/backend/src/metabase_enterprise/stale/api.clj b/enterprise/backend/src/metabase_enterprise/stale/api.clj
index 6d630b46dc1..c08c56fb9c5 100644
--- a/enterprise/backend/src/metabase_enterprise/stale/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/stale/api.clj
@@ -63,7 +63,9 @@
(map (fn [card]
(-> card
(assoc :model (if (queries/model? card) "dataset" "card"))
- (dissoc :dataset_query))))))
+ ;; Selected so `queries/model?` above can read :type and so the Card schema upgrade
+ ;; can run; none of them are part of the listing payload.
+ (dissoc :dataset_query :type :result_metadata :dimensions :dimension_mappings))))))
(defn- annotate-dashboard-with-collection-info
"For dashboards, we want `here` and `location` since they can contain cards as children."
diff --git a/enterprise/backend/src/metabase_enterprise/stale/db.clj b/enterprise/backend/src/metabase_enterprise/stale/db.clj
index dd9a412384e..b738f39fac4 100644
--- a/enterprise/backend/src/metabase_enterprise/stale/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/stale/db.clj
@@ -35,6 +35,12 @@
[nil :location]
:dataset_query
:card_schema
+ ;; :type is read by `queries/model?` when labelling the row; the rest are here only
+ ;; because :dataset_query is schema-governed and the Card schema upgrade needs them.
+ :type
+ :result_metadata
+ :dimensions
+ :dimension_mappings
:last_used_at
[^:allow-subquery
{:select [:status]
diff --git a/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj b/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj
index d39f89f0264..6bd675254c7 100644
--- a/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj
@@ -66,7 +66,7 @@
(is (not= 0 (t2/count :model/Card {:where [:= :database_id audit/audit-db-id]}))
"Cards should be created for Audit DB when the content is there."))
(testing "Cards in the audit collection have non-empty :result_metadata after installation"
- (let [audit-cards (t2/select [:model/Card :id :name :result_metadata :card_schema]
+ (let [audit-cards (t2/select [:model/Card :id :name :result_metadata :card_schema :type :database_id :dataset_query :dimensions :dimension_mappings]
:database_id audit/audit-db-id)
audit-cards-no-metadata (filter (comp empty? :result_metadata) audit-cards)]
(is (seq audit-cards))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj
index 616878c654d..df360d407f0 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj
@@ -130,7 +130,7 @@
:to_entity_id child-id}]
(is (= #{8}
(t2/select-fn-set (comp count :result_metadata)
- [:model/Card :id :result_metadata :card_schema]
+ [:model/Card :id :result_metadata :card_schema :type :database_id :dataset_query :dimensions :dimension_mappings]
:id [:in [parent-id child-id grandchild-id]])))
(t2/update! :model/Card parent-id {:dataset_query (lib/query mp orders)})
(mt/with-dynamic-fn-redefs [async/submit! (fn [f] (f))]
@@ -140,7 +140,7 @@
:user-id api/*current-user-id*}))
(is (= #{9}
(t2/select-fn-set (comp count :result_metadata)
- [:model/Card :id :result_metadata :card_schema]
+ [:model/Card :id :result_metadata :card_schema :type :database_id :dataset_query :dimensions :dimension_mappings]
:id [:in [parent-id child-id grandchild-id]])))))))))
(deftest ^:synchronized native-card-update-does-not-update-children-test
@@ -165,7 +165,7 @@
:user-id api/*current-user-id*}))
(is (= nil
(t2/select-one-fn :result_metadata
- [:model/Card :id :result_metadata :card_schema]
+ [:model/Card :id :result_metadata :card_schema :type :database_id :dataset_query :dimensions :dimension_mappings]
:id child-id)))))))))
(deftest ^:synchronized model-update-passes-down-new-values-test
@@ -201,7 +201,7 @@
:user-id api/*current-user-id*}))
(is (= #{[child-id "new-name"] [grandchild-id "new-name"]}
(t2/select-fn-set (juxt :id #(get-in % [:result_metadata 0 :display_name]))
- [:model/Card :id :result_metadata :card_schema]
+ [:model/Card :id :result_metadata :card_schema :type :database_id :dataset_query :dimensions :dimension_mappings]
:id [:in [child-id grandchild-id]]))))))))))
(deftest ^:synchronized model-update-respects-child-overrides-test
@@ -244,7 +244,7 @@
:user-id api/*current-user-id*}))
(is (= #{[child-id "child-name"] [grandchild-id "grandchild-name"]}
(t2/select-fn-set (juxt :id #(get-in % [:result_metadata 0 :display_name]))
- [:model/Card :id :result_metadata :card_schema]
+ [:model/Card :id :result_metadata :card_schema :type :database_id :dataset_query :dimensions :dimension_mappings]
:id [:in [child-id grandchild-id]]))))))))))
(deftest ^:synchronized model-update-stops-recursing-when-child-metadata-is-unchanged-test
@@ -286,7 +286,7 @@
:user-id api/*current-user-id*}))
(is (= nil
(t2/select-one-fn #(get-in % [:result_metadata 0 :display_name])
- [:model/Card :id :result_metadata :card_schema]
+ [:model/Card :id :result_metadata :card_schema :type :database_id :dataset_query :dimensions :dimension_mappings]
:id grandchild-id))))))))))
(defn- with-syncable-db! [thunk]
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/database_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/database_test.clj
index 292f39110ce..6acca116dba 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/database_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/database_test.clj
@@ -20,7 +20,10 @@
metabase-enterprise.sandbox.api.table-test/native-query-metadata-test."
[group]
(let [card (t2/select-one :model/Card
- {:select [:c.id :c.dataset_query :c.entity_id :c.card_schema]
+ {:select [:c.id :c.dataset_query :c.entity_id :c.card_schema
+ ;; required alongside :card_schema for the Card schema upgrade
+ :c.type :c.database_id :c.result_metadata
+ :c.dimensions :c.dimension_mappings]
:from [[:sandboxes :s]]
:join [[:permissions_group :pg] [:= :s.group_id :pg.id]
[:report_card :c] [:= :c.id :s.card_id]]
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj
index e8d2009395f..0465f71477b 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/table_test.clj
@@ -69,7 +69,10 @@
:attributes {:cat 50}}
;; Fetch the card and manually compute & save the metadata
(let [card (t2/select-one :model/Card
- {:select [:c.id :c.dataset_query :c.entity_id :c.card_schema]
+ {:select [:c.id :c.dataset_query :c.entity_id :c.card_schema
+ ;; required alongside :card_schema for the Card schema upgrade
+ :c.type :c.database_id :c.result_metadata
+ :c.dimensions :c.dimension_mappings]
:from [[:sandboxes :s]]
:join [[:permissions_group :pg] [:= :s.group_id :pg.id]
[:report_card :c] [:= :c.id :s.card_id]]
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
index a37fe531661..f0a8c144c40 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
@@ -963,7 +963,11 @@
[:field %products.category {:join-alias "Products"}]]}]})}]
;; Populate the native source card's result_metadata the way the app does when a user runs and
;; saves the query. This is the state serdes must preserve across the round-trip.
- (let [source-cols (-> (qp/process-query (t2/select-one-fn :dataset_query [:model/Card :dataset_query] native-id))
+ (let [source-cols (-> (qp/process-query (t2/select-one-fn :dataset_query
+ [:model/Card :id :dataset_query :card_schema
+ :type :database_id :result_metadata
+ :dimensions :dimension_mappings]
+ native-id))
(get-in [:data :results_metadata :columns]))
source-names (mapv :name source-cols)]
(t2/update! :model/Card native-id {:result_metadata source-cols})
diff --git a/src/metabase/actions/db.clj b/src/metabase/actions/db.clj
index 168a82e42f8..b197c03b3ef 100644
--- a/src/metabase/actions/db.clj
+++ b/src/metabase/actions/db.clj
@@ -4,6 +4,7 @@
(:require
[metabase.actions.schema :as actions.schema]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.queries.card-schema :as queries.card-schema]
[metabase.queries.schema :as queries.schema]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
@@ -47,7 +48,7 @@
(mu/defn card-type
"The type of the Card with `card-id`, or nil."
[card-id :- ::lib.schema.id/card]
- (t2/select-one-fn :type [:model/Card :type :card_schema] :id card-id))
+ (t2/select-one-fn :type [:model/Card :type] :id card-id))
(mu/defn table
"The Table with `table-id`, or nil."
@@ -224,9 +225,12 @@
:where [:in :action.id action-ids]}))
(mu/defn card-scope-columns
- "The query, Collection id, Database id, and display of the Card with `card-id`, or nil."
+ "The query-relevant columns of the Card with `card-id`, plus its Collection id and display, or nil."
[card-id :- ::lib.schema.id/card]
- (t2/select-one [:model/Card :dataset_query :collection_id :database_id :display] card-id))
+ ;; Spelled out rather than calling `metabase.queries.core/card-query-info`: `queries` loads `actions` (via
+ ;; `driver-api`), so requiring its API namespace back from here is a cyclic load. `queries.card-schema` is the
+ ;; dependency-free namespace that exists for exactly this.
+ (t2/select-one (queries.card-schema/selection [:collection_id :database_id :display]) :id card-id))
(mu/defn dashboard-collection-id
"The Collection id of the Dashboard with `dashboard-id`, or nil."
diff --git a/src/metabase/activity_feed/db.clj b/src/metabase/activity_feed/db.clj
index a2a96071872..88b34bd7ca0 100644
--- a/src/metabase/activity_feed/db.clj
+++ b/src/metabase/activity_feed/db.clj
@@ -17,6 +17,8 @@
(t2/select [:model/Card
:id :name :collection_id :description :display
:dataset_query :type :archived :card_schema
+ ;; required alongside :card_schema for the Card schema upgrade
+ :result_metadata :dimensions :dimension_mappings
:collection.authority_level [:collection.name :collection_name]
[:dashboard.name :dashboard_name] :dashboard_id]
{:where [:in :report_card.id ids]
@@ -211,6 +213,10 @@
:card.database_id
:card.display
:card.card_schema
+ ;; required alongside :card_schema for the Card schema upgrade
+ :card.type
+ :card.dimensions
+ :card.dimension_mappings
:card.result_metadata
:card.dataset_query
:card.entity_id
diff --git a/src/metabase/analytics/db.clj b/src/metabase/analytics/db.clj
index 66265c9322f..f679e4c617e 100644
--- a/src/metabase/analytics/db.clj
+++ b/src/metabase/analytics/db.clj
@@ -149,9 +149,9 @@
(t2/count :model/Collection {:where (mi/exclude-internal-content-hsql :model/Collection)}))
(mu/defn card-collection-ids
- "The Collection id and schema of the non-internal Cards."
+ "The Collection id of the non-internal Cards."
[]
- (t2/select [:model/Card :collection_id :card_schema] {:where [:and (mi/exclude-internal-content-hsql :model/Card)]}))
+ (t2/select [:model/Card :collection_id] {:where [:and (mi/exclude-internal-content-hsql :model/Card)]}))
(mu/defn database-stats-columns
"The sync, engine, and DBMS version of the non-internal Databases."
diff --git a/src/metabase/audit_app/db.clj b/src/metabase/audit_app/db.clj
index c6ff7f815e1..40ed57777ac 100644
--- a/src/metabase/audit_app/db.clj
+++ b/src/metabase/audit_app/db.clj
@@ -32,7 +32,7 @@
(mu/defn card-name-and-description
"The name and description of the Card with `card-id`, or nil."
[card-id :- ::lib.schema.id/card]
- (t2/select-one [:model/Card :name :description :card_schema], :id card-id))
+ (t2/select-one [:model/Card :name :description], :id card-id))
(mu/defn table-database-id
"The Database id of the Table with `table-id`, or nil."
diff --git a/src/metabase/dashboards/db.clj b/src/metabase/dashboards/db.clj
index 76f8ff78f0a..3a2c3e7bec3 100644
--- a/src/metabase/dashboards/db.clj
+++ b/src/metabase/dashboards/db.clj
@@ -8,6 +8,7 @@
[metabase.dashboards.schema :as dashboards.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.models.serialization :as serdes]
+ [metabase.queries.core :as queries]
[metabase.queries.schema :as queries.schema]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
@@ -118,9 +119,9 @@
(t2/select-one :model/Card :id card-id))
(mu/defn card-query-columns
- "The query and schema of the Card with `card-id`, or nil."
+ "The query-relevant columns of the Card with `card-id`, or nil."
[card-id :- ::lib.schema.id/card]
- (t2/select-one [:model/Card :dataset_query :card_schema] :id card-id))
+ (queries/card-query-info card-id))
(mu/defn card-queries
"A map of Card id to query for the Cards with `card-ids`."
@@ -167,7 +168,8 @@
"The series Cards of the DashboardCards with `dashcard-ids`, each with its `:dashboardcard_id`, in series order."
[dashcard-ids :- [:sequential ::lib.schema.id/dashcard]]
(t2/select [:model/Card :id :name :description :display :dataset_query :type :database_id
- :visualization_settings :collection_id :card_schema :series.dashboardcard_id]
+ :visualization_settings :collection_id :card_schema :result_metadata
+ :dimensions :dimension_mappings :series.dashboardcard_id]
{:left-join [[:dashboardcard_series :series] [:= :report_card.id :series.card_id]]
:where [:in :series.dashboardcard_id dashcard-ids]
:order-by [[:series.position :asc]]}))
diff --git a/src/metabase/dashboards/models/dashboard_card.clj b/src/metabase/dashboards/models/dashboard_card.clj
index ed6d7350149..d4a38767129 100644
--- a/src/metabase/dashboards/models/dashboard_card.clj
+++ b/src/metabase/dashboards/models/dashboard_card.clj
@@ -139,7 +139,12 @@
dashcard-id->series (when (seq dashcard-ids)
(as-> (dashboards.db/series-cards-for-dashcards dashcard-ids) series
(group-by :dashboardcard_id series)
- (update-vals series #(map (fn [card] (dissoc card :dashboardcard_id)) %))))]
+ ;; :result_metadata, :dimensions and :dimension_mappings are selected only so the
+ ;; Card schema upgrade can run; they are not part of the series payload.
+ (update-vals series #(map (fn [card]
+ (dissoc card :dashboardcard_id :result_metadata
+ :dimensions :dimension_mappings))
+ %))))]
(map (fn [dashcard]
(assoc dashcard :series (get dashcard-id->series (:id dashcard) [])))
dashcards))))
diff --git a/src/metabase/explorations/db.clj b/src/metabase/explorations/db.clj
index 316da830ffd..43871ddbc44 100644
--- a/src/metabase/explorations/db.clj
+++ b/src/metabase/explorations/db.clj
@@ -12,6 +12,7 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.queries.core :as queries]
[metabase.queries.schema :as queries.schema]
+ [metabase.util :as u]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]))
@@ -770,17 +771,14 @@
(t2/select-one [:model/Card :name :description :display :visualization_settings] :id card-id))
(mu/defn card-queries
- "The ID, schema, Database, and query of the Cards with `card-ids`."
+ "The query-relevant columns of the Cards with `card-ids`."
[card-ids :- [:sequential ::lib.schema.id/card]]
- (t2/select [:model/Card :id :card_schema :database_id :dataset_query] :id [:in card-ids]))
+ (queries/cards-queries-info card-ids :include [:database_id]))
(mu/defn metric-cards-by-id
"A map of ID to the planner columns of the Cards with `card-ids`."
[card-ids :- [:sequential ::lib.schema.id/card]]
- (t2/select-pk->fn identity
- [:model/Card :id :name :description :database_id :dataset_query :card_schema :dimensions
- :dimension_mappings]
- :id [:in card-ids]))
+ (u/index-by :id (queries/cards-queries-info card-ids :include [:name :description :database_id])))
(mu/defn metric-card-ids
"The `:id`s of the Cards visible to the current user as metrics, restricted to `metric-ids` when
@@ -800,7 +798,7 @@
;;; the response stays small and JSON encoding is fast.
(def ^:private exploration-card-columns
[:id :name :description :collection_id :database_id :table_id :type :entity_id
- :card_schema :dataset_query :dimensions :dimension_mappings])
+ :card_schema :dataset_query :result_metadata :dimensions :dimension_mappings])
(mu/defn metric-cards-for-explorations
"The exploration-relevant columns of the metric Cards with `card-ids`."
diff --git a/src/metabase/explorations/derived_perms.clj b/src/metabase/explorations/derived_perms.clj
index d9d75df94f6..04827c11b17 100644
--- a/src/metabase/explorations/derived_perms.clj
+++ b/src/metabase/explorations/derived_perms.clj
@@ -112,7 +112,6 @@
[(:exploration_thread_id b) cid])))
blocks)
cards (when (seq card-id)
- ;; `:card_schema` is mandatory in any explicit Card column list
(u/index-by :id (explorations.db/card-queries (distinct (vals card-id)))))]
(map (fn [{thread-id :id :as thread}]
(let [card (get cards (get card-id thread-id))]
diff --git a/src/metabase/explorations/query_plan/context.clj b/src/metabase/explorations/query_plan/context.clj
index 94de330f2b2..85c28ebabfc 100644
--- a/src/metabase/explorations/query_plan/context.clj
+++ b/src/metabase/explorations/query_plan/context.clj
@@ -187,8 +187,8 @@
...]}
The underlying Card is hydrated with the columns the variant builders need
- (`:id :name :description :database_id :dataset_query :card_schema :dimensions
- :dimension_mappings`), once per Card even when it appears in several blocks."
+ (`:id :name :description :database_id :dataset_query :dimensions :dimension_mappings`),
+ once per Card even when it appears in several blocks."
[blocks]
(let [card-ids (distinct (mapcat #(map :card_id (:metrics %)) blocks))
cards (when (seq card-ids)
diff --git a/src/metabase/lib_be/db.clj b/src/metabase/lib_be/db.clj
index 10f95eee526..6e252a84ac7 100644
--- a/src/metabase/lib_be/db.clj
+++ b/src/metabase/lib_be/db.clj
@@ -9,9 +9,9 @@
[toucan2.core :as t2]))
(mu/defn card-database-ids
- "The `:id`, `:database_id`, and `:card_schema` of the Cards with `card-ids`."
+ "The `:id` and `:database_id` of the Cards with `card-ids`."
[card-ids :- [:set ::lib.schema.id/card]]
- (t2/select [:model/Card :id :database_id :card_schema] :id [:in card-ids]))
+ (t2/select [:model/Card :id :database_id] :id [:in card-ids]))
(mu/defn database
"The `:metadata/database` with `database-id`, or nil."
diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.clj
index 01754c81cce..4e6993ea392 100644
--- a/src/metabase/lib_be/metadata/jvm.clj
+++ b/src/metabase/lib_be/metadata/jvm.clj
@@ -339,9 +339,13 @@
(next-method query-type model parsed-args honeysql)
{:select [:card/collection_id
:card/created_at ; Needed for backfilling :entity_id on demand; see [[metabase.queries.models.card]].
- :card/card_schema ; Needed for after-select logic to work.
+ ;; :card_schema and every column the Card schema upgrade reads must be selected together,
+ ;; or `upgrade-card-schema-to-latest` cannot run. See [[metabase.queries.models.card]].
+ :card/card_schema
:card/database_id
:card/dataset_query
+ :card/dimensions
+ :card/dimension_mappings
:card/id
:card/entity_id
:card/name
diff --git a/src/metabase/mcp/v2/tools/browse.clj b/src/metabase/mcp/v2/tools/browse.clj
index ac9ee001d02..bb99b6caf72 100644
--- a/src/metabase/mcp/v2/tools/browse.clj
+++ b/src/metabase/mcp/v2/tools/browse.clj
@@ -222,11 +222,11 @@
"Columns `list_models` selects for the `:question` projection."
;; Naming the columns keeps `t2/select` from running the `dataset_query` and `result_metadata`
;; transforms on every model — the expensive part of listing a database with thousands of cards.
- ;; `:card_schema` is never projected, but Card's after-select hook throws without it once the
- ;; row carries `:id` plus any of `:dataset_query`/`:result_metadata`/`:database_id`/`:type`.
+ ;; None of these are [[metabase.queries.card-schema/schema-governed-columns]], so the Card schema
+ ;; upgrade stays out of the way and `:card_schema` does not have to ride along.
;; `question-enrichment-keys` are projection keys `get_content` computes, not Card columns;
;; selecting them would name a nonexistent column, so they are dropped from the DB select here.
- (into [:model/Card :card_schema]
+ (into [:model/Card]
(remove projections/question-enrichment-keys projections/question-detailed-keys)))
(defn- list-models
diff --git a/src/metabase/metabot/db.clj b/src/metabase/metabot/db.clj
index 57c66c524f2..07369a8cd9b 100644
--- a/src/metabase/metabot/db.clj
+++ b/src/metabase/metabot/db.clj
@@ -692,9 +692,9 @@
(t2/select-one :model/Card :id card-id :type type))
(mu/defn card-type-row
- "The ID, type, and schema of the Card with `card-id`, or nil."
+ "The ID and type of the Card with `card-id`, or nil."
[card-id :- ::lib.schema.id/card]
- (t2/select-one [:model/Card :id :type :card_schema] :id card-id))
+ (t2/select-one [:model/Card :id :type] :id card-id))
(defn- metabot-metrics-and-models-query
"Honey SQL query selecting the metric and model Cards in scope of the Metabot with `metabot-id` that are visible to
@@ -783,26 +783,26 @@
(mu/defn card-search-rows
"The searchable columns of the Cards with `card-ids`."
[card-ids :- [:sequential ::lib.schema.id/card]]
- (t2/select [:model/Card :id :name :description :database_id :collection_id :card_schema :type] :id [:in card-ids]))
+ (t2/select [:model/Card :id :name :description :database_id :collection_id :type] :id [:in card-ids]))
(mu/defn unarchived-card-summaries
"The presentable columns of the unarchived Cards with `card-ids`."
[card-ids :- [:set ::lib.schema.id/card]]
- (t2/select [:model/Card :id :name :type :description :card_schema :collection_id :database_id :table_id]
+ (t2/select [:model/Card :id :name :type :description :collection_id :database_id :table_id]
:id [:in card-ids]
:archived false))
(mu/defn cards-in-collection
"The presentable columns of the unarchived Cards in the Collection with `collection-id`, ordered by name."
[collection-id :- ::lib.schema.id/collection]
- (t2/select [:model/Card :id :name :type :description :card_schema :collection_id :database_id :table_id]
+ (t2/select [:model/Card :id :name :type :description :collection_id :database_id :table_id]
{:where [:and [:= :collection_id collection-id] [:= :archived false]]
:order-by [[:%lower.name :asc]]}))
(mu/defn cards-for-table
"The presentable columns of the unarchived Cards on the Table with `table-id`, ordered by name."
[table-id :- ::lib.schema.id/table]
- (t2/select [:model/Card :id :name :type :description :card_schema :collection_id :database_id :table_id]
+ (t2/select [:model/Card :id :name :type :description :collection_id :database_id :table_id]
:table_id table-id
:archived false
{:order-by [[:%lower.name :asc]]}))
@@ -810,7 +810,7 @@
(mu/defn models-for-database
"The presentable columns of the unarchived model Cards on the Database with `database-id`, ordered by name."
[database-id :- ::lib.schema.id/database]
- (t2/select [:model/Card :id :name :type :description :card_schema :collection_id :database_id :table_id]
+ (t2/select [:model/Card :id :name :type :description :collection_id :database_id :table_id]
:type :model
:database_id database-id
:archived false
diff --git a/src/metabase/metrics/core.clj b/src/metabase/metrics/core.clj
index 5527a8fec49..29bd4a34583 100644
--- a/src/metabase/metrics/core.clj
+++ b/src/metabase/metrics/core.clj
@@ -256,7 +256,7 @@
(defn sync-metric-dimensions-for-database!
"Compute and persist dimensions for every metric Card in `database-id` that doesn't have any yet."
[database-id]
- (doseq [{:keys [id dimensions]} (metrics.db/metric-cards-for-database database-id)
+ (doseq [{:keys [id dimensions]} (metrics.db/raw-metric-cards-for-database database-id)
:when (empty? dimensions)]
(try
(sync-dimensions! :metadata/metric id)
diff --git a/src/metabase/metrics/db.clj b/src/metabase/metrics/db.clj
index 2b383b46d9f..f6b05d6a2fb 100644
--- a/src/metabase/metrics/db.clj
+++ b/src/metabase/metrics/db.clj
@@ -47,10 +47,14 @@
[id :- ::lib.schema.id/measure]
(t2/select-one :model/Measure :id id))
-(mu/defn metric-cards-for-database
- "The id and dimensions of the metric Cards of the Database with `database-id`."
+(mu/defn raw-metric-cards-for-database
+ "The id and dimensions of the metric Cards of the Database with `database-id`.
+
+ Selects the `report_card` table directly rather than `:model/Card`: this asks whether dimensions have ever been
+ *persisted*, so it has to see the stored column. Going through the model runs the `:card_schema` upgrades, which
+ populates missing `:dimensions`."
[database-id :- ::lib.schema.id/database]
- (t2/select [:model/Card :id :dimensions] :type "metric" :database_id database-id))
+ (t2/select [:report_card :id :dimensions] :type "metric" :database_id database-id))
(mu/defn fields-with-columns
"The id and `columns` of the Fields with `field-ids`."
diff --git a/src/metabase/model_persistence/db.clj b/src/metabase/model_persistence/db.clj
index 34645a3dd66..17569a498fa 100644
--- a/src/metabase/model_persistence/db.clj
+++ b/src/metabase/model_persistence/db.clj
@@ -241,7 +241,7 @@
(mu/defn card-archived-and-type
"The archived flag and type of the Card with `card-id`, or nil."
[card-id :- ::lib.schema.id/card]
- (t2/select-one [:model/Card :archived :type :card_schema] :id card-id))
+ (t2/select-one [:model/Card :archived :type] :id card-id))
(mu/defn database
"The Database with `database-id`, or nil."
diff --git a/src/metabase/models/db.clj b/src/metabase/models/db.clj
index 675c9058d28..ed625f5a540 100644
--- a/src/metabase/models/db.clj
+++ b/src/metabase/models/db.clj
@@ -340,9 +340,9 @@
(t2/select :metadata/table :db_id database-id :schema schema :name table-name))
(mu/defn card-serdes-columns
- "The id, entity id, Collection id, Database id, and schema of the Card with `card-id`, or nil."
+ "The id, entity id, Collection id, and Database id of the Card with `card-id`, or nil."
[card-id :- ::lib.schema.id/card]
- (t2/select-one [:model/Card :id :entity_id :collection_id :database_id :card_schema] :id card-id))
+ (t2/select-one [:model/Card :id :entity_id :collection_id :database_id] :id card-id))
(mu/defn measure-serdes-columns
"The id, entity id, and Table id of the Measure with `measure-id`, or nil."
diff --git a/src/metabase/models/serialization/resolve/mp.clj b/src/metabase/models/serialization/resolve/mp.clj
index 7b949b4a776..5eee4d85f00 100644
--- a/src/metabase/models/serialization/resolve/mp.clj
+++ b/src/metabase/models/serialization/resolve/mp.clj
@@ -345,8 +345,6 @@
(when (int? card-id)
;; `api/read-check` for Cards needs only the parent collection. Avoid loading and
;; transforming the entire dataset_query just to export one stable identifier.
- ;; `:card_schema` must ride along: selecting `:database_id` makes the after-select
- ;; treat this as a full card row and demand it.
(models.db/card-serdes-columns card-id)))
(measure-by-id [_ measure-id]
(when measure-id
diff --git a/src/metabase/public_sharing_rest/db.clj b/src/metabase/public_sharing_rest/db.clj
index 19630e15fc3..7e917edfa6f 100644
--- a/src/metabase/public_sharing_rest/db.clj
+++ b/src/metabase/public_sharing_rest/db.clj
@@ -15,7 +15,7 @@
[card-id :- [:maybe ::lib.schema.id/card]
& {:keys [enable-embedding?]} :- [:maybe [:map {:closed true} [:enable-embedding? {:optional true} [:maybe :boolean]]]]]
(t2/select-one [:model/Card :id :dataset_query :description :display :name :parameters :visualization_settings
- :card_schema]
+ :card_schema :type :result_metadata :dimensions :dimension_mappings]
{:where [:and
[:= :id card-id]
[:= :archived false]
diff --git a/src/metabase/pulse/db.clj b/src/metabase/pulse/db.clj
index 01f0f7a90b6..32154a459d3 100644
--- a/src/metabase/pulse/db.clj
+++ b/src/metabase/pulse/db.clj
@@ -17,7 +17,13 @@
(mu/defn card-query
"The query of the Card with `card-id`, or nil."
[card-id :- ::lib.schema.id/card]
- (t2/select-one-fn :dataset_query [:model/Card :dataset_query] card-id))
+ ;; Written out by hand rather than using `metabase.queries.core/card-query-info`: `queries` already uses
+ ;; `pulse`, so requiring it back would add a module cycle for one query. Keep in sync with
+ ;; `metabase.queries.card-schema/schema-upgrade-triggers`.
+ (t2/select-one-fn :dataset_query
+ [:model/Card :id :dataset_query :card_schema :type
+ :result_metadata :dimensions :dimension_mappings]
+ card-id))
(mu/defn dashboard
"The Dashboard with `dashboard-id`, or nil. `dashboard-id` may be nil (e.g. a legacy Pulse with no Dashboard), in
diff --git a/src/metabase/queries/card_schema.clj b/src/metabase/queries/card_schema.clj
new file mode 100644
index 00000000000..6b26d0b3e3e
--- /dev/null
+++ b/src/metabase/queries/card_schema.clj
@@ -0,0 +1,48 @@
+(ns metabase.queries.card-schema
+ "The `:model/Card` columns that have to be SELECTed together for the `:card_schema` upgrade to run, plus the
+ helpers that project them.
+
+ This namespace is deliberately tiny. `metabase.queries.core` pulls in most of the `queries` module, and the
+ low-level `db.clj` namespaces that need these helpers sit *below* it in the load graph — requiring the module's
+ main API namespace from them is a cyclic load. Nothing here needs more than Toucan.")
+
+(def schema-governed-columns
+ "The columns of `:model/Card` whose stored representation is relevant to `:card_schema`. These columns must be
+ read together, and will be written back together.
+
+ Reading any one of them is what arms the upgrade: a projection that names none of them has nothing an upgrade
+ could change, so it is free to SELECT whatever narrow set of columns it likes.
+
+ Keep this in sync when adding an upgrade that rewrites a new column."
+ #{:dataset_query :result_metadata :dimensions :dimension_mappings})
+
+(def schema-upgrade-triggers
+ "Every column a [[metabase.queries.models.card/upgrade-card-schema-to]] implementation reads or writes. A SELECT
+ naming any [[schema-governed-columns]] must include all of these, or the upgrade cannot run correctly.
+
+ Beyond the governed columns themselves this is:
+
+ - `:card_schema`, the version marker that says which upgrades still have to run. It is not a governed
+ representation, so selecting it alone must not demand the rest of the set.
+ - `:type`, read by the upgrade to 24 to recognize a metric. It is never written by an upgrade, but leaving it out
+ is worse than an error: the upgrade silently skips the card and still stamps it as current.
+
+ Keep this in sync with the columns read by the `upgrade-card-schema-to` implementations."
+ (conj schema-governed-columns :card_schema :type))
+
+(def ^:private schema-select-columns
+ "The `[modelable & columns]` projection [[selection]] SELECTs with: `:id`, so callers can key the rows they get
+ back, plus every [[schema-upgrade-triggers]] column."
+ (into [:id] schema-upgrade-triggers))
+
+(def ^:private base-selection
+ (into [:model/Card] schema-select-columns))
+
+(defn selection
+ "Returns a `[:model/Card columns...]` projection with all the query-related fields (see [[schema-upgrade-triggers]])
+ plus `:id` and similar.
+
+ Accepts an optional seq of extra columns to include."
+ ([] base-selection)
+ ([extras]
+ (into base-selection extras)))
diff --git a/src/metabase/queries/core.clj b/src/metabase/queries/core.clj
index aa40dd77f4e..2d97f74d73b 100644
--- a/src/metabase/queries/core.clj
+++ b/src/metabase/queries/core.clj
@@ -3,6 +3,7 @@
[metabase.queries.cached-result]
[metabase.queries.card]
[metabase.queries.card-write-checks]
+ [metabase.queries.db]
[metabase.queries.metadata]
[metabase.queries.models.card]
[metabase.queries.models.card.metadata]
@@ -30,6 +31,9 @@
card-param-constraints
card-param-values
card-param-remapped-value]
+ [metabase.queries.db
+ card-query-info
+ cards-queries-info]
[metabase.queries.models.card
create-card!]
[metabase.queries.card-write-checks
diff --git a/src/metabase/queries/db.clj b/src/metabase/queries/db.clj
index a6a0cb09ab7..45d31b645c5 100644
--- a/src/metabase/queries/db.clj
+++ b/src/metabase/queries/db.clj
@@ -7,6 +7,7 @@
[metabase.dashboards.schema :as dashboards.schema]
[metabase.lib-be.schema :as lib-be.schema]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.queries.card-schema :as queries.card-schema]
[metabase.queries.schema :as queries.schema]
[metabase.query-processor.schema]
[metabase.util.honey-sql-2 :as h2x]
@@ -63,14 +64,19 @@
(t2/select :model/Card :id [:in card-ids]))
(mu/defn card-query-info
- "The query, type, result metadata, and schema of the Card with `card-id`."
- [card-id :- ::lib.schema.id/card]
- (t2/select-one [:model/Card :dataset_query :type :result_metadata :card_schema] :id card-id))
+ "The query-related fields of the Card with `card-id`.
+
+ Accepts kv-args:
+ - `:include` is a seq of extra columns to select."
+ [card-id :- ::lib.schema.id/card
+ & {:keys [include]} :- [:maybe [:map {:closed true}
+ [:include {:optional true} [:seqable :keyword]]]]]
+ (t2/select-one (queries.card-schema/selection include) :id card-id))
(mu/defn card-dataset-query
"The `:dataset_query` of the Card with `card-id`."
[card-id :- ::lib.schema.id/card]
- (t2/select-one-fn :dataset_query [:model/Card :dataset_query :card_schema] :id card-id))
+ (:dataset_query (card-query-info card-id)))
(mu/defn card-document-id
"The `:document_id` of the Card with `card-id`."
@@ -82,7 +88,7 @@
[card-id :- ::lib.schema.id/card]
(t2/select-one-fn :parameters [:model/Card :parameters] :id card-id))
-(mu/defn card-dimensions
+(mu/defn raw-card-dimensions
"The raw `:dimensions` row of the Card with `card-id`."
[card-id :- ::lib.schema.id/card]
(t2/query-one {:select [:dimensions]
@@ -94,16 +100,20 @@
[card-id :- ::lib.schema.id/card]
(t2/select-one [:model/Card [:database_id :database-id] [:table_id :table-id]] :id card-id))
-(mu/defn card-queries
- "The IDs and queries of the Cards with `card-ids`."
- [card-ids :- [:sequential ::lib.schema.id/card]]
- (t2/select [:model/Card :id :dataset_query :card_schema] :id [:in card-ids]))
+(mu/defn cards-queries-info
+ "The IDs and query-related fields of the Cards with `card-ids`.
+
+ Accepts optional kv-args:
+ - `:include` a seq of extra columns to select."
+ [card-ids :- [:seqable ::lib.schema.id/card]
+ & {:keys [include]} :- [:maybe [:map {:closed true}
+ [:include {:optional true} [:seqable :keyword]]]]]
+ (t2/select (queries.card-schema/selection include) :id [:in card-ids]))
(mu/defn mbql-model-cards
- "The IDs, types, queries, and result metadata of the unarchived MBQL models that query the Database with
- `database-id`."
+ "The unarchived MBQL models that query the Database with `database-id`."
[database-id :- ::lib.schema.id/database]
- (t2/select [:model/Card :id :type :dataset_query :result_metadata :card_schema]
+ (t2/select (queries.card-schema/selection)
:database_id database-id
:type :model
:query_type :query
@@ -112,7 +122,7 @@
(mu/defn source-card-dependents
"The IDs and source Card IDs of the Cards whose source Card is one of `source-card-ids`."
[source-card-ids :- [:or [:set ::lib.schema.id/card] [:sequential ::lib.schema.id/card]]]
- (t2/select [:model/Card :id :source_card_id :card_schema] :source_card_id [:in source-card-ids]))
+ (t2/select [:model/Card :id :source_card_id] :source_card_id [:in source-card-ids]))
(mu/defn metric-cards-for-source-cards
"The unarchived metric Cards built on one of `source-card-ids`, ordered by name."
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 5de79e6addf..eb0795dca8a 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -2,6 +2,7 @@
"Underlying DB model for what is now most commonly referred to as a 'Question' in most user-facing situations. Card
is a historical name, but is the same thing; both terms are used interchangeably in the backend codebase."
(:require
+ [better-cond.core :as b]
[clojure.set :as set]
[honey.sql.helpers :as sql.helpers]
[medley.core :as m]
@@ -32,6 +33,7 @@
[metabase.premium-features.core :refer [defenterprise]]
[metabase.public-sharing.core :as public-sharing]
[metabase.pulse.core :as pulse]
+ [metabase.queries.card-schema :as card-schema]
[metabase.queries.db :as queries.db]
[metabase.queries.models.card.metadata :as card.metadata]
[metabase.queries.models.parameter-card :as parameter-card]
@@ -111,7 +113,7 @@
(defmethod metrics/dimensions-initialized? :metadata/metric
[metric]
- (some? (:dimensions (queries.db/card-dimensions (:id metric)))))
+ (some? (:dimensions (queries.db/raw-card-dimensions (:id metric)))))
(t2/deftransforms :model/Card
{:dataset_query lib-be/transform-query
@@ -546,6 +548,13 @@
(when-let [action-ids (queries.db/implicit-action-ids-for-model model-id)]
(queries.db/delete-actions! action-ids)))
+#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
+(def ^:private ^:dynamic *update-baseline-select?*
+ "True when the select is being for the purposes of fetching the current baseline before diffing it in
+ [[t2/update!]]. This can be checked during [[t2/define-after-select]] to restrict certain logic to run only for
+ true, top-level `t2/select*` calls and not for the internal reads during an update."
+ false)
+
;;; TODO (Cam 7/21/25) -- icky to have some of the before-update stuff live in the before-update method below and then
;;; some but not all of it live in this `pre-update` function... all of the before-update stuff should live in a single
;;; function. We should move it all here or move it all there. Weird to split it up between two places.
@@ -614,9 +623,15 @@
Metadata-provider fetches must be skipped: nothing consumes `:query_description` on lib metadata, and computing it
resolves the metric's own `:metric` refs through the metadata provider, whose card fetches re-enter this
after-select — with a cycle in the metric reference graph the recursion is unbounded and overflows the stack
- (#74954)."
+ (#74954).
+
+ Column-restricted SELECTs are skipped too. `:query_description` is derived, not a column, so a caller that named
+ its columns cannot have been asking for it — and building the Lib query to compute it is expensive. This matters
+ because [[card-schema/schema-upgrade-triggers]] forces `:type` into every projection that reads a schema-governed column,
+ which would otherwise opt all of them in to a field none of them read."
[card]
(if-not (and (map? card)
+ (empty? (:columns t2.pipeline/*parsed-args*))
(= :metric (:type card))
(not (metadata-provider-fetch? card))
(-> card :dataset_query not-empty)
@@ -676,33 +691,90 @@
(assoc card :dimensions dimensions :dimension_mappings dimension-mappings))
card))
-(defn- plausible-card-select?
- "Whether `card` looks like it was SELECTed as a real Card row (as opposed to some sort of odd query, like an
- aggregation over cards, that happens to run through the same after-select hook)."
+(defn- schema-governed-select?
+ "Whether `card` is a Card row SELECTed with any [[card-schema/schema-governed-columns]], and therefore holds a
+ stored representation an upgrade could rewrite.
+
+ A projection naming none of them — `[:id :name]`, `[:id :type :display]`, a `COUNT(*)` over `report_card` that
+ happens to run through this same after-select hook — has nothing to upgrade, so it is left alone and may SELECT
+ as narrow a set of columns as it likes. Anything that does name one must bring the rest of
+ [[card-schema/schema-upgrade-triggers]] along.
+
+ `:id` is required as well, and not as a formality: Card *revisions* are Card-shaped maps that go through this
+ same after-select, and `metabase.revisions.impl.card` deliberately drops `:id`, `:dimensions`, and
+ `:dimension_mappings` from the snapshot it stores. Those are historical records rather than live rows, so they
+ are left alone here; upgrading one would mean recomputing a dimension set for a card as it looked in the past."
[card]
(boolean (and (:id card)
- (or (:dataset_query card)
- (:result_metadata card)
- (:database_id card)
- (:type card)))))
+ (some #(contains? card %) card-schema/schema-governed-columns))))
(mu/defn- upgrade-card-schema-to-latest :- ::queries.schema/card
- "Run the schema upgrades over a plausible Card row and normalize it. Only call this
- when [[plausible-card-select?]] is true; other queries against `:report_card` should be returned as-is."
- [card :- ::queries.schema/card]
- (-> (if-not (:card_schema card)
- ;; Plausible but no :card_schema - error.
- (throw (ex-info "Cannot SELECT a Card without including :card_schema"
- {:card-id (:id card)}))
- ;; Plausible and has the schema, so run the upgrades over it.
- (loop [card card]
- ;; Use >= to allow for downgrades.
- (if (>= (:card_schema card) current-schema-version)
- card
- (let [new-version (inc (:card_schema card))]
- (recur (assoc (upgrade-card-schema-to card new-version)
- :card_schema new-version))))))
- (->> (lib/normalize ::queries.schema/card))))
+ "Run the schema upgrades over a Card row and normalize it."
+ ([card :- ::queries.schema/card]
+ (upgrade-card-schema-to-latest card *update-baseline-select?*))
+ ([card :- ::queries.schema/card
+ update-baseline-select? :- :boolean]
+ (->> (b/cond
+ ;; Nothing an upgrade could rewrite was SELECTed, so there is nothing to do and nothing to demand.
+ ;; `after-select` checks this too, to keep odd `report_card` queries out of here entirely.
+ (not (schema-governed-select? card))
+ card
+
+ ;; Fetching some of the "hot" parts of a card but not all of them - error!
+ :let [missing (remove #(contains? card %) card-schema/schema-upgrade-triggers)]
+ (seq missing)
+ (throw (ex-info "Cannot SELECT a Card with card_schema columns without all columns needed for an upgrade"
+ {:card-id (:id card)
+ :missing missing}))
+
+ ;; Skip the schema updates when the select is being done as the baseline for an UPDATE.
+ update-baseline-select?
+ card
+
+ ;; All gates passed, so run the schema upgrades.
+ :else
+ (loop [card card]
+ ;; Use >= to allow for downgrades.
+ (if (>= (:card_schema card) current-schema-version)
+ card
+ (let [new-version (inc (:card_schema card))]
+ (recur (assoc (upgrade-card-schema-to card new-version)
+ :card_schema new-version))))))
+ (lib/normalize ::queries.schema/card))))
+
+(defn- migrate-schema-governed-columns
+ "Cards are upgraded to the current schema on read, in memory only.
+
+ The vital invariant is that `:card_schema`, `:dataset_query`, `:dimensions`, and the other
+ [[card-schema/schema-governed-columns]] are all upgraded together.
+
+ The superset [[card-schema/schema-upgrade-triggers]] includes other fields which are *read* during schema upgrades, without
+ themselves being updated in lockstep. If any of these change, then all the [[card-schema/schema-upgrade-triggers]] should be
+ updated too. Of course any incoming writes to e.g. `:dataset_query` will win over upgrading the old value.
+
+ Unrelated writes, e.g. to `:name` or `:collection_id`, don't cause the [[card-schema/schema-governed-columns]] to be updated."
+ [card original changes]
+ (let [stored-schema (:card_schema original)]
+ (if-not (and stored-schema
+ ;; Use < to leave downgraded rows alone, matching [[upgrade-card-schema-to-latest]].
+ (< stored-schema current-schema-version)
+ (some #(contains? changes %) card-schema/schema-upgrade-triggers))
+ card
+ ;; Force the `update-baseline-select?` flag to false, since this runs inside the `t2/before-update` process.
+ ;; The schema upgrades are suppressed during `t2/update!` so that when it reads the baseline it sees the
+ ;; [[card-schema/schema-governed-columns]] as they are really stored, not post-upgrade. That's important for the `t2/update!`
+ ;; diff process.
+ ;; However, here we really do want to run the upgrades, so we can write back a card which is fully updated in
+ ;; lockstep.
+ (let [upgraded (upgrade-card-schema-to-latest
+ (merge original changes {:card_schema (:card_schema original)})
+ false)
+ ;; Keep the `upgraded` version of any [[card-schema/schema-governed-columns]], but prefer those in `changes`.
+ columns (remove (set (keys changes)) card-schema/schema-governed-columns)
+ relevant (when (seq columns)
+ (select-keys upgraded columns))]
+ (-> (merge card relevant)
+ (assoc :card_schema current-schema-version))))))
(defonce ^:private unique-cards-with-blank-dataset-query
(atom #{}))
@@ -735,7 +807,8 @@
;; changes, which likely indicates a bug.
(methodical/defmethod t2.pipeline/results-transform [:toucan.result-type/instances :model/Card]
[query-type model]
- (let [xform (next-method query-type model)]
+ (let [xform (next-method query-type model)
+ baseline? (isa? query-type :toucan.query-type/select.instances.from-update)]
(fn xform' [rf]
(let [rf' (xform rf)]
(fn rf''
@@ -743,8 +816,9 @@
([acc]
(rf' acc))
([acc card]
- (lib/with-card-clean-hook (partial mbql5-conversion-clean-callback card)
- (rf' acc card))))))))
+ (binding [*update-baseline-select?* baseline?]
+ (lib/with-card-clean-hook (partial mbql5-conversion-clean-callback card)
+ (rf' acc card)))))))))
(t2/define-after-select :model/Card
[card]
@@ -759,7 +833,7 @@
public-sharing/remove-public-uuid-if-public-sharing-is-disabled
add-query-description-to-metric-card
;; At this point, the card should be at schema version 20 or higher.
- (cond-> (plausible-card-select? card) upgrade-card-schema-to-latest)
+ (cond-> (schema-governed-select? card) upgrade-card-schema-to-latest)
monitor-blank-dataset-query))
(defn- normalize-card
@@ -829,12 +903,15 @@
(t2/define-before-update :model/Card
[{:keys [verified-result-metadata?] :as card}]
- (let [changes (some-> card t2/changes normalize-card)
- card (normalize-card card)]
+ (let [changes (some-> card t2/changes normalize-card)
+ ;; Captured before `normalize-card` rebinds `card`, so the migration below never depends on
+ ;; normalization preserving the instance's original.
+ original (t2/original card)
+ card (normalize-card card)]
(collection/check-allowed-content (:type card) (:collection_id changes))
(-> card
(dissoc :verified-result-metadata?)
- (assoc :card_schema current-schema-version)
+ (migrate-schema-governed-columns original changes)
(apply-dashboard-question-updates changes)
(m/update-existing :dataset_query lib-be/normalize-query)
(populate-result-metadata changes verified-result-metadata?)
@@ -1191,7 +1268,7 @@
A card's `database_id` comes from its source card, so updating a dependent first preserves the old ID."
[root-card-id old-db-id]
(when-let [all-dep-ids (seq (graph/transitive (->SourceCardDependentsGraph) [root-card-id]))]
- (let [id->card (m/index-by :id (queries.db/card-queries all-dep-ids))]
+ (let [id->card (m/index-by :id (queries.db/cards-queries-info all-dep-ids))]
(into []
(comp (map id->card)
(filter #(= (get-in % [:dataset_query :database]) old-db-id)))
diff --git a/src/metabase/queries_rest/db.clj b/src/metabase/queries_rest/db.clj
index d388a87f650..840e520d96f 100644
--- a/src/metabase/queries_rest/db.clj
+++ b/src/metabase/queries_rest/db.clj
@@ -63,14 +63,14 @@
:order-by [[[:lower :c.name] :asc]]}))
(mu/defn public-cards
- "The name, id, public uuid, and schema of the unarchived Cards that are publicly shared."
+ "The name, id, and public uuid of the unarchived Cards that are publicly shared."
[]
- (t2/select [:model/Card :name :id :public_uuid :card_schema], :public_uuid [:not= nil], :archived false))
+ (t2/select [:model/Card :name :id :public_uuid], :public_uuid [:not= nil], :archived false))
(mu/defn embeddable-cards
- "The name, id, and schema of the unarchived Cards with embedding enabled."
+ "The name and id of the unarchived Cards with embedding enabled."
[]
- (t2/select [:model/Card :name :id :card_schema], :enable_embedding true, :archived false))
+ (t2/select [:model/Card :name :id], :enable_embedding true, :archived false))
(mu/defn segment-database-id
"The Database id of the Table of the Segment with `segment-id`, or nil."
@@ -101,9 +101,9 @@
(t2/select-one-fn :dataset_query :model/Card :id card-id))
(mu/defn card-public-uuid-columns
- "The public uuid and schema of the Card with `card-id`, or nil."
+ "The public uuid of the Card with `card-id`, or nil."
[card-id :- ::lib.schema.id/card]
- (t2/select-one [:model/Card :public_uuid :card_schema] :id card-id))
+ (t2/select-one [:model/Card :public_uuid] :id card-id))
(mu/defn compatible-series-cards
"The unarchived Cards other than `excluded-card-id` displayed as one of `display-types`, newest first, whose id is
@@ -158,7 +158,8 @@
`new-collection-id-or-nil`."
[card-ids :- [:set ::lib.schema.id/card]
new-collection-id-or-nil :- [:maybe ms/PositiveInt]]
- (t2/select [:model/Card :id :collection_id :collection_position :dataset_query :card_schema]
+ (t2/select [:model/Card :id :collection_id :collection_position :dataset_query :card_schema :type
+ :result_metadata :dimensions :dimension_mappings]
{:where [:and [:in :id card-ids]
[:or [:not= :collection_id new-collection-id-or-nil]
(when new-collection-id-or-nil
diff --git a/src/metabase/query_permissions/db.clj b/src/metabase/query_permissions/db.clj
index 91a00465d70..49066ffd23d 100644
--- a/src/metabase/query_permissions/db.clj
+++ b/src/metabase/query_permissions/db.clj
@@ -8,9 +8,9 @@
[toucan2.core :as t2]))
(mu/defn card-collection-id
- "The `:collection_id` and `:card_schema` of the Card with `card-id`, or nil."
+ "The `:collection_id` of the Card with `card-id`, or nil."
[card-id :- ::lib.schema.id/card]
- (t2/select-one [:model/Card :collection_id :card_schema] :id card-id))
+ (t2/select-one [:model/Card :collection_id] :id card-id))
(mu/defn card-not-in-database
"The Card with `card-id` if it does not belong to the Database with `database-id`, or nil."
diff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.clj
index b1c81c78bc4..738e54c70a4 100644
--- a/src/metabase/query_processor/api.clj
+++ b/src/metabase/query_processor/api.clj
@@ -79,7 +79,7 @@
(let [source-card-id (when (= (lib/normalized-query-type query) :mbql/query)
(query->source-card-id query))
source-card (when source-card-id
- (query-processor.db/source-card-metadata source-card-id))
+ (queries/card-query-info source-card-id))
info (cond-> {:executed-by api/*current-user-id*
:context context
:card-id source-card-id}
diff --git a/src/metabase/query_processor/db.clj b/src/metabase/query_processor/db.clj
index cf4d6ba412f..e2db6da1554 100644
--- a/src/metabase/query_processor/db.clj
+++ b/src/metabase/query_processor/db.clj
@@ -21,11 +21,6 @@
[table-id :- [:maybe ::lib.schema.id/table]]
(t2/select-one :model/Table :id table-id {:from [(warehouse-schema-overlay/table-query)]}))
-(mu/defn source-card-metadata
- "The entity id, result metadata, and type of the Card with `card-id`, or nil."
- [card-id :- ::lib.schema.id/card]
- (t2/select-one [:model/Card :entity_id :result_metadata :type :card_schema] :id card-id))
-
(mu/defn dashcard-series-exists?
"Whether the Card with `card-id` is a series of the DashboardCard with `dashcard-id`."
[card-id :- [:maybe ::lib.schema.id/card]
@@ -85,9 +80,9 @@
:updated_at :updated_at}}))
(mu/defn card-database-ids
- "The `:id`, `:database_id`, and `:card_schema` of the Cards with `card-ids`."
+ "The `:id` and `:database_id` of the Cards with `card-ids`."
[card-ids :- [:set ::lib.schema.id/card]]
- (t2/select [:model/Card :id :database_id :card_schema] :id [:in card-ids]))
+ (t2/select [:model/Card :id :database_id] :id [:in card-ids]))
(mu/defn upsert-cache-entry!
"Insert or update the QueryCache entry for `query-hash`, setting `:results` to `results` and `:updated_at` to
diff --git a/src/metabase/search/db.clj b/src/metabase/search/db.clj
index 04fb30472f0..8a777db24ba 100644
--- a/src/metabase/search/db.clj
+++ b/src/metabase/search/db.clj
@@ -5,6 +5,7 @@
[honey.sql.helpers :as sql.helpers]
[metabase.app-db.core :as mdb]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.models.interface :as mi]
[metabase.search.appdb.index-schema :as index-schema]
[metabase.search.appdb.query :as appdb.query]
[metabase.search.appdb.scoring :as search.scoring]
@@ -393,6 +394,17 @@
(t2/select-pk->fn :common_name [:model/User :id :first_name :last_name :email] :id [:in user-ids]))
(mu/defn card-result-metadata
- "A map of Card id to result metadata for the Cards with `card-ids`."
+ "A map of Card id to result metadata for the Cards with `card-ids`.
+
+ SELECTs `report_card` directly instead of `:model/Card`, so the `:card_schema` read-time upgrade stays out of it.
+ Naming a schema-governed column on the model arms the whole upgrade chain, and the upgrade to 24 recomputes a
+ legacy metric's entire dimension set from its query — several app-db round trips per row, to produce columns
+ search never looks at. What search wants is the stored result metadata, so all that is needed from the model is
+ its `:out` transform, applied here. (The upgrade to 22 also strips `:ident` keys left by an abandoned experiment;
+ a stale one reaching a search payload is harmless.)"
[card-ids :- [:set ::lib.schema.id/card]]
- (t2/select-pk->fn :result_metadata [:model/Card :id :card_schema :result_metadata] :id [:in card-ids]))
+ (let [result-metadata-out (:out mi/transform-result-metadata)]
+ (into {}
+ (map (juxt :id (comp result-metadata-out :result_metadata)))
+ (when (seq card-ids)
+ (t2/select [(t2/table-name :model/Card) :id :result_metadata] :id [:in card-ids])))))
diff --git a/src/metabase/search/impl.clj b/src/metabase/search/impl.clj
index dc9806f3929..a601c5823b6 100644
--- a/src/metabase/search/impl.clj
+++ b/src/metabase/search/impl.clj
@@ -460,9 +460,7 @@
(filter #(contains? #{"card" "metric" "dataset"} (:model %)))
(map :id))
search-results)
- card-metadata (if (empty? card-ids)
- {}
- (search.db/card-result-metadata card-ids))]
+ card-metadata (search.db/card-result-metadata card-ids)]
(map (fn [{:keys [model id] :as item}]
(if (contains? #{"card" "metric" "dataset"} model)
(assoc item :result_metadata (card-metadata id))
diff --git a/src/metabase/typed_schemas/db.clj b/src/metabase/typed_schemas/db.clj
index 27769a18b37..4bc54f2c376 100644
--- a/src/metabase/typed_schemas/db.clj
+++ b/src/metabase/typed_schemas/db.clj
@@ -4,6 +4,7 @@
(:require
[metabase.collections.models.collection :as collection]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.queries.core :as queries]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
@@ -45,9 +46,10 @@
(t2/select [:model/Field :id :table_id] :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn card-dimensions
- "The dimensions and dimension mappings of the Card with `card-id`, or nil."
+ "The query-relevant columns (`:dimensions` and `:dimension_mappings` among them) of the Card with `card-id`,
+ or nil."
[card-id :- ::lib.schema.id/card]
- (t2/select-one [:model/Card :dimensions :dimension_mappings] :id card-id))
+ (queries/card-query-info card-id))
(mu/defn table-names
"The id, database id, name, and display name of the Tables with `table-ids`."
diff --git a/src/metabase/upload/db.clj b/src/metabase/upload/db.clj
index 82c963bb4ce..257081b6d53 100644
--- a/src/metabase/upload/db.clj
+++ b/src/metabase/upload/db.clj
@@ -3,6 +3,7 @@
additional logic, so the rest of the module only touches `toucan2.core` for hydration."
(:require
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.queries.card-schema :as queries.card-schema]
[metabase.queries.schema :as queries.schema]
[metabase.util.malli :as mu]
[metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
@@ -91,17 +92,20 @@
(t2/update! :model/Field field-id {:display_name display-name}))
(mu/defn unarchived-models-for-table
- "The id, query, and schema of the unarchived model Cards of the Table with `table-id`."
+ "The query-relevant columns of the unarchived model Cards of the Table with `table-id`."
[table-id :- ::lib.schema.id/table]
- (t2/select [:model/Card :id :dataset_query :card_schema]
+ (t2/select (queries.card-schema/selection)
:table_id table-id
:type :model
:archived false))
(mu/defn card-query-and-metadata
- "The query, result metadata, and schema of the Card with `card-id`, or nil."
+ "The query-relevant columns of the Card with `card-id`, or nil."
[card-id :- ::lib.schema.id/card]
- (t2/select-one [:model/Card :dataset_query :result_metadata :card_schema] card-id))
+ ;; Spelled out rather than calling `metabase.queries.core/card-query-info`: `queries` loads `upload` (via
+ ;; `driver-api`), so requiring its API namespace back from here is a cyclic load. `queries.card-schema` is the
+ ;; dependency-free namespace that exists for exactly this.
+ (t2/select-one (queries.card-schema/selection) :id card-id))
(mu/defn set-card-result-metadata!
"Set the result metadata of the Card with `card-id`."
diff --git a/src/metabase/usage_metadata/db.clj b/src/metabase/usage_metadata/db.clj
index 50db33a4b02..587fc61bd4d 100644
--- a/src/metabase/usage_metadata/db.clj
+++ b/src/metabase/usage_metadata/db.clj
@@ -245,4 +245,5 @@
(mu/defn unarchived-metric-cards
"The id, Database id, query, and schema of the unarchived metric Cards."
[]
- (t2/select [:model/Card :id :database_id :dataset_query :card_schema] :type "metric" :archived false))
+ (t2/select [:model/Card :id :database_id :dataset_query :card_schema :type :result_metadata :dimensions :dimension_mappings]
+ :type "metric" :archived false))
diff --git a/src/metabase/warehouse_schema/db.clj b/src/metabase/warehouse_schema/db.clj
index 3961f99af85..67837624114 100644
--- a/src/metabase/warehouse_schema/db.clj
+++ b/src/metabase/warehouse_schema/db.clj
@@ -577,6 +577,8 @@
{:select [:c.id :c.dataset_query :c.result_metadata :c.name
:c.description :c.collection_id :c.database_id :c.type
:c.source_card_id :c.created_at :c.entity_id :c.card_schema
+ ;; required alongside :card_schema for the Card schema upgrade
+ :c.dimensions :c.dimension_mappings
[:r.status :moderated_status]]
:from [[:report_card :c]]
:left-join [[^:allow-subquery {:select [:moderated_item_id :status]
diff --git a/src/metabase/warehouses_rest/db.clj b/src/metabase/warehouses_rest/db.clj
index 27aa200e0da..a841f1a208d 100644
--- a/src/metabase/warehouses_rest/db.clj
+++ b/src/metabase/warehouses_rest/db.clj
@@ -49,8 +49,11 @@
[card-type :- [:or :keyword :string]
database-ids :- [:set ::lib.schema.id/database]
collection-scope :- [:maybe [:or [:= :root] [:set ::lib.schema.id/collection] [:sequential ::lib.schema.id/collection]]]]
+ ;; Rows are pushed through `mi/do-after-select :model/Card` by the caller, so `:dataset_query` and
+ ;; `:result_metadata` drag in the rest of [[metabase.queries.card-schema/schema-upgrade-triggers]].
(t2/reducible-query {:select [:name :description :database_id :dataset_query :id :collection_id
:result_metadata :type :source_card_id :card_schema
+ :dimensions :dimension_mappings
[^:allow-subquery {:select [:status]
:from [:moderation_review]
:where [:and
@@ -175,7 +178,7 @@
search-card-slug :- :string
include-dashboard-questions? :- [:maybe :boolean]]
(t2/select [:model/Card :id :type :database_id :name :collection_id
- [:collection.name :collection_name] :card_schema]
+ [:collection.name :collection_name]]
{:where [:and
[:= :report_card.database_id database-id]
[:= :report_card.archived false]
diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.clj
index 7b644fb349b..1c6b126804e 100644
--- a/test/metabase/queries/models/card_test.clj
+++ b/test/metabase/queries/models/card_test.clj
@@ -13,6 +13,7 @@
[metabase.lib.test-util.notebook-helpers :as notebook-helpers]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
+ [metabase.queries.card-schema :as card-schema]
[metabase.queries.db :as queries.db]
[metabase.queries.models.card :as card]
[metabase.queries.models.parameter-card :as parameter-card]
@@ -866,14 +867,40 @@
:visualization_settings
json/decode+kw))))))
+(def ^:private complete-card
+ "A card carrying every column the schema upgrade could need. Any SELECT that reads a schema-governed
+ column must project all of these — see [[metabase.queries.card-schema/schema-upgrade-triggers]]."
+ {:id 1
+ :type :question
+ :database_id 1
+ :dataset_query {}
+ :result_metadata nil
+ :dimensions nil
+ :dimension_mappings nil})
+
(deftest ^:parallel upgrade-card-schema-after-downgrade
- (testing "We exit the loop if a chard_schema is higher than the current schema."
- (let [card {:id 1
- :dataset_query {}
- :card_schema (inc @#'card/current-schema-version)}]
+ (testing "We exit the loop if a card_schema is higher than the current schema."
+ (let [card (assoc complete-card :card_schema (inc @#'card/current-schema-version))]
(is (= card
(#'card/upgrade-card-schema-to-latest card))))))
+(deftest ^:parallel upgrade-card-schema-requires-every-trigger-column-test
+ (testing (str "Reading a schema-governed column without the rest of the columns an upgrade needs is a "
+ "programming error in the query, not a condition to recover from at read time. It throws "
+ "unconditionally — including when this row happens to be current — so the bug surfaces on "
+ "every instance rather than only the ones holding an out-of-date card.")
+ (doseq [omitted (disj card-schema/schema-upgrade-triggers :card_schema)]
+ (testing (str "omitting " omitted)
+ (let [card (dissoc (assoc complete-card :card_schema @#'card/current-schema-version) omitted)]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Cannot SELECT a Card with card_schema columns without all columns needed for an upgrade"
+ (#'card/upgrade-card-schema-to-latest card))
+ "a current-schema row is rejected just the same, so the missing column is never latent")))))
+ (testing "a card with no schema-governed column at all is passed through untouched"
+ (let [card {:id 1 :name "no governed columns here"}]
+ (is (= card (#'card/upgrade-card-schema-to-latest card))))))
+
(deftest storing-metabase-version
(testing "Newly created Card should know a Metabase version used to create it"
(mt/with-temp [:model/Card card {}]
@@ -1217,12 +1244,43 @@
(is (= "Orders, Metric A"
(:query_description (t2/select-one :model/Card :id b-id))))))))
+(defn- stored-card-schema
+ "Read `card_schema` straight out of `report_card`. A `t2/select-one` would run the after-select schema
+ upgrade and report the current version whether or not it was ever written."
+ [card-id]
+ (:card_schema (t2/query-one {:select [:card_schema]
+ :from [:report_card]
+ :where [:= :id card-id]})))
+
+(defn- age-card!
+ "Put `card-id`'s stored `card_schema` behind the current version. Needs a raw UPDATE because
+ before-insert forces `:card_schema` to current, so passing it to `with-temp` is silently ignored."
+ [card-id schema-version]
+ (t2/query-one {:update :report_card
+ :set {:card_schema schema-version}
+ :where [:= :id card-id]})
+ (assert (= schema-version (stored-card-schema card-id))))
+
(deftest before-update-card-schema-test
- (testing "card_schema gets set to current-schema-version on update"
- (mt/with-temp [:model/Card {card-id :id} {:card_schema 20}]
- (t2/update! :model/Card card-id {:name "Updated Name"})
- (is (= @#'card/current-schema-version
- (t2/select-one-fn :card_schema :model/Card :id card-id))))))
+ (testing "card_schema moves as a unit with the columns it governs"
+ (testing "an update touching no schema-governed column leaves the vintage alone"
+ (mt/with-temp [:model/Card {card-id :id} {}]
+ (age-card! card-id 20)
+ (t2/update! :model/Card card-id {:name "Updated Name"})
+ (is (= "Updated Name" (t2/select-one-fn :name :model/Card :id card-id))
+ "the update itself is persisted")
+ (is (= 20 (stored-card-schema card-id))
+ (str "stamping the current version here would strand the row: the columns the schema governs "
+ "still hold their old representation, and upgrade-card-schema-to-latest would never run "
+ "over them again."))))
+ (testing "an update touching a schema-governed column migrates the group and stamps the version"
+ (mt/with-temp [:model/Card {card-id :id} {}]
+ (age-card! card-id 20)
+ (t2/update! :model/Card card-id {:dataset_query (mt/mbql-query venues)})
+ (is (= @#'card/current-schema-version (stored-card-schema card-id))
+ (str "the bump must reach the database. t2/update! diffs changes against a baseline SELECT; if "
+ "the after-select schema upgrade runs on that baseline it already reads as current, the "
+ "diff drops :card_schema, and the row stays behind forever."))))))
(deftest before-update-dashboard-question-updates-test
(testing "apply-dashboard-question-updates is called"
@@ -1713,8 +1771,8 @@
:model/Card question4 (dependent-card db1-id question2)
:model/Card question5 (dependent-card db1-id question4)]
;; H2 returns these rows source-first. Reverse them to cover a valid result order from PostgreSQL.
- (mt/with-dynamic-fn-redefs [queries.db/card-queries
- (comp reverse (mt/original-fn #'queries.db/card-queries))]
+ (mt/with-dynamic-fn-redefs [queries.db/cards-queries-info
+ (comp reverse (mt/original-fn #'queries.db/cards-queries-info))]
(mt/with-test-user :crowberto
(card/update-card! {:card-before-update model
:card-updates {:dataset_query {:lib/type :mbql/query
@@ -1803,3 +1861,16 @@
(is (not (contains? extracted (eid summary-card)))
"a card belonging to an exploration Summary is never exported — its name and dataset_query
carry values discovered under the creator's lens, and its parent document is excluded")))))
+
+(deftest before-update-schema-upgrade-uses-incoming-query-test
+ (testing "a query edit to a legacy metric persists dimensions derived from the new query, not the old one"
+ (mt/with-temp [:model/Card {card-id :id} {:type :metric
+ :database_id (mt/id)
+ :table_id (mt/id :venues)
+ :dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]
+ (t2/query-one {:update :report_card
+ :set {:card_schema 23, :dimensions nil, :dimension_mappings nil}
+ :where [:= :id card-id]})
+ (t2/update! :model/Card card-id {:dataset_query (mt/mbql-query categories {:aggregation [[:count]]})})
+ (is (= #{(mt/id :categories)}
+ (into #{} (map :table-id) (t2/select-one-fn :dimension_mappings :model/Card :id card-id)))))))
diff --git a/test/metabase/search/db_test.clj b/test/metabase/search/db_test.clj
new file mode 100644
index 00000000000..1b1148cd390
--- /dev/null
+++ b/test/metabase/search/db_test.clj
@@ -0,0 +1,45 @@
+(ns metabase.search.db-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.search.db :as search.db]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn- metric-query
+ "A metric-style query over `venues`: one aggregation, no breakouts."
+ []
+ (let [mp (mt/metadata-provider)]
+ (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))
+ (lib/aggregate (lib/count)))))
+
+(deftest card-result-metadata-does-not-run-the-card-schema-upgrade-test
+ (testing "reading result metadata for a legacy metric costs one query"
+ (mt/with-temp [:model/Card {card-id :id} {:type :metric
+ :database_id (mt/id)
+ :table_id (mt/id :venues)
+ :dataset_query (metric-query)}]
+ ;; Put the metric back in the un-curated state the upgrade to 24 looks for. A raw UPDATE, because
+ ;; before-insert forces `:card_schema` to current and `with-temp` would silently ignore it.
+ (t2/query-one {:update :report_card
+ :set {:card_schema 23, :dimensions nil, :dimension_mappings nil}
+ :where [:= :id card-id]})
+ (is (= ["count"]
+ (t2/with-call-count [call-count]
+ (let [metadata (get (search.db/card-result-metadata #{card-id}) card-id)]
+ (is (= 1 (call-count))
+ (str "Going through `:model/Card` here would arm the `:card_schema` upgrade to 24, which "
+ "recomputes this metric's whole dimension set from its query — several more queries "
+ "per row, for columns search throws away."))
+ (mapv :name metadata))))))))
+
+(deftest ^:parallel card-result-metadata-no-cards-test
+ (testing "no card ids means no query, and no `IN ()` for the app DB to choke on"
+ (is (= {}
+ (t2/with-call-count [call-count]
+ (let [result (search.db/card-result-metadata #{})]
+ (is (zero? (call-count)))
+ result))))))