Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[MBQL lib] Refactor a common pattern into new lib functions (#68474)

Braden Shepherdson39b4a2183023
diff --git a/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj b/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj
index 56c506bfbce..865274708d9 100644
--- a/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/agent_api/api_test.clj
@@ -325,7 +325,7 @@
(let [decoded (decode-query response)]
(is (= :mbql/query (lib/normalized-query-type decoded)))
(is (= (mt/id) (lib/database-id decoded)))
- (is (= (mt/id :orders) (lib/source-table-id decoded))))))
+ (is (= (mt/id :orders) (lib/primary-source-table-id decoded))))))
(testing "Applies default limit of 100 when no limit is specified"
(let [table-id (mt/id :orders)
diff --git a/src/metabase/actions/scope.clj b/src/metabase/actions/scope.clj
index a3132305f0f..d882443f37e 100644
--- a/src/metabase/actions/scope.clj
+++ b/src/metabase/actions/scope.clj
@@ -29,7 +29,7 @@
(if (and (contains? scope :collection-id) (contains? scope :table-id) (contains? scope :database-id))
scope
(let [card (t2/select-one [:model/Card :dataset_query :collection_id :database_id :display] card-id)
- table-id (lib/source-table-id (:dataset_query card))]
+ table-id (lib/primary-source-table-id (:dataset_query card))]
(merge {:table-id table-id
:collection-id (:collection_id card missing-id)
:database-id (:database_id card missing-id)}
diff --git a/src/metabase/lib/card.cljc b/src/metabase/lib/card.cljc
index 7f307de06af..9cec97621f4 100644
--- a/src/metabase/lib/card.cljc
+++ b/src/metabase/lib/card.cljc
@@ -313,9 +313,7 @@
(mu/defn source-card-type :- [:maybe ::lib.schema.metadata/card.type]
"The type of the query's source-card, if it has one."
[query :- ::lib.schema/query]
- (when-let [card-id (lib.util/source-card-id query)]
- (when-let [card (lib.metadata/card query card-id)]
- (:type card))))
+ (some-> query lib.metadata.calculation/primary-source-card :type))
(mu/defn source-card-is-model? :- :boolean
"Is the query's source-card a model?"
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index d11eac45891..c9f84734907 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -52,6 +52,7 @@
[metabase.lib.ref :as lib.ref]
[metabase.lib.remove-replace :as lib.remove-replace]
[metabase.lib.schema]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.util]
[metabase.lib.segment :as lib.segment]
[metabase.lib.serialize]
@@ -64,6 +65,7 @@
[metabase.lib.util.unique-name-generator]
[metabase.lib.validate :as lib.validate]
[metabase.lib.walk.util]
+ [metabase.util.malli :as mu]
[metabase.util.namespaces :as shared.ns]))
(comment lib.aggregation/keep-me
@@ -506,8 +508,6 @@
previous-stage
previous-stage-number
query-stage
- source-table-id
- source-card-id
update-query-stage]
[metabase.lib.util.unique-name-generator
non-truncating-unique-name-generator
@@ -548,3 +548,29 @@
[hook-fn & body]
`(binding [lib.convert/*card-clean-hook* ~hook-fn]
~@body)))
+
+(mu/defn primary-source-table-id :- [:maybe ::lib.schema.id/table]
+ "If the first stage of `a-query` is an MBQL stage with a `:source-table`, return that table ID. For a native stage or
+ a `:source-card`, returns nil.
+
+ Prefer [[primary-source-table]] instead, when you want the `:metadata/table` rather than just its ID.
+
+ **DO NOT** use this for permissions - this is only the *primary* source, not a complete list of table IDs required
+ by `a-query`.
+
+ **Code Health:** Discouraged; there are few legitimate use cases for working with raw table IDs outside lib."
+ [a-query :- :metabase.lib.schema/query]
+ (lib.util/source-table-id a-query))
+
+(mu/defn primary-source-card-id :- [:maybe ::lib.schema.id/card]
+ "If the first stage of `a-query` is an MBQL stage with a `:source-card`, return that card ID. For a native stage or
+ a `:source-table`, returns nil.
+
+ Prefer [[primary-source-card]] instead, when you want the `:metadata/card` rather than just its ID.
+
+ **DO NOT** use this for permissions - this is only the *primary* source, not a complete list of card IDs required
+ by `a-query`.
+
+ **Code Health:** Discouraged; there are few legitimate use cases for working with raw card IDs outside lib."
+ [a-query :- :metabase.lib.schema/query]
+ (lib.util/source-card-id a-query))
diff --git a/src/metabase/lib/database.cljc b/src/metabase/lib/database.cljc
index e6e0bf53509..8accea70fa0 100644
--- a/src/metabase/lib/database.cljc
+++ b/src/metabase/lib/database.cljc
@@ -1,9 +1,8 @@
(ns metabase.lib.database
(:require
- [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.metadata.calculation :as lib.metadata.calculation]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
- [metabase.lib.util :as lib.util]
[metabase.util.malli :as mu]))
(mu/defn database-id :- [:maybe ::lib.schema.id/database]
@@ -19,6 +18,4 @@
(when-let [id (:database query)]
(if (not= id lib.schema.id/saved-questions-virtual-database-id)
id
- (when-let [source-card-id (lib.util/source-card-id query)]
- (when-let [card-metadata (lib.metadata/card query source-card-id)]
- (:database-id card-metadata))))))
+ (some-> query lib.metadata.calculation/primary-source-card :database-id))))
diff --git a/src/metabase/lib/drill_thru/fk_filter.cljc b/src/metabase/lib/drill_thru/fk_filter.cljc
index 52e01aa2c74..5266f3fe879 100644
--- a/src/metabase/lib/drill_thru/fk_filter.cljc
+++ b/src/metabase/lib/drill_thru/fk_filter.cljc
@@ -25,7 +25,6 @@
[metabase.lib.aggregation :as lib.aggregation]
[metabase.lib.drill-thru.common :as lib.drill-thru.common]
[metabase.lib.filter :as lib.filter]
- [metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.calculation :as lib.metadata.calculation]
[metabase.lib.options :as lib.options]
[metabase.lib.schema :as lib.schema]
@@ -51,16 +50,14 @@
(lib.drill-thru.common/mbql-stage? query stage-number)
(not (lib.types.isa/primary-key? column))
(lib.types.isa/foreign-key? column))
- (let [source (or (some->> query lib.util/source-table-id (lib.metadata/table query))
- (some->> query lib.util/source-card-id (lib.metadata/card query)))
- filter-expr (if (= value :null)
+ (let [filter-expr (if (= value :null)
(lib.options/ensure-uuid (lib.filter/is-null column-ref))
(lib.options/ensure-uuid (lib.filter/= column-ref value)))]
{:lib/type :metabase.lib.drill-thru/drill-thru
:type :drill-thru/fk-filter
:filter filter-expr
:column-name (lib.metadata.calculation/display-name query stage-number column :long)
- :table-name (lib.metadata.calculation/display-name query 0 source)})))
+ :table-name (lib.metadata.calculation/display-name query 0 (lib.metadata.calculation/primary-source query))})))
(defmethod lib.drill-thru.common/drill-thru-info-method :drill-thru/fk-filter
[_query _stage-number drill-thru]
diff --git a/src/metabase/lib/drill_thru/underlying_records.cljc b/src/metabase/lib/drill_thru/underlying_records.cljc
index a9bb1a1418f..c96ed9bf386 100644
--- a/src/metabase/lib/drill_thru/underlying_records.cljc
+++ b/src/metabase/lib/drill_thru/underlying_records.cljc
@@ -37,7 +37,6 @@
[metabase.lib.fe-util :as lib.fe-util]
[metabase.lib.filter :as lib.filter]
[metabase.lib.join :as lib.join]
- [metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.calculation :as lib.metadata.calculation]
[metabase.lib.options :as lib.options]
[metabase.lib.schema :as lib.schema]
@@ -112,9 +111,8 @@
(not (neg? value)))
value
2)
- :table-name (when-let [table-or-card (or (some->> query lib.util/source-table-id (lib.metadata/table query))
- (some->> query lib.util/source-card-id (lib.metadata/card query)))]
- (lib.metadata.calculation/display-name query stage-number table-or-card))
+ :table-name (some->> (lib.metadata.calculation/primary-source query)
+ (lib.metadata.calculation/display-name query stage-number))
:dimensions traceable-dimensions
;; If the underlying column comes from an aggregation, then the column-ref needs to be updated as well to the
;; corresponding aggregation ref so that [[drill-underlying-records]] knows to extract the filter implied by
diff --git a/src/metabase/lib/expression.cljc b/src/metabase/lib/expression.cljc
index 35d2ce7f679..6dde4e6b756 100644
--- a/src/metabase/lib/expression.cljc
+++ b/src/metabase/lib/expression.cljc
@@ -72,12 +72,11 @@
(when <>
(log/warnf "Found expression %s in previous stage" (pr-str expression-name)))))
(when (lib.util/first-stage? query stage-number)
- (when-let [source-card-id (lib.util/source-card-id query)]
- (when-let [source-card (lib.metadata/card query source-card-id)]
- (u/prog1 (resolve-expression (:dataset-query source-card) expression-name)
- (when <>
- (log/warnf "Found expression %s in source card %d. Next time, use a :field name ref!"
- (pr-str expression-name) source-card-id))))))
+ (when-let [source-card (lib.metadata.calculation/primary-source-card query)]
+ (u/prog1 (resolve-expression (:dataset-query source-card) expression-name)
+ (when <>
+ (log/warnf "Found expression %s in source card %d. Next time, use a :field name ref!"
+ (pr-str expression-name) (:id source-card))))))
(throw (ex-info (i18n/tru "No expression named {0}" (pr-str expression-name))
{:expression-name expression-name
:query query
diff --git a/src/metabase/lib/join.cljc b/src/metabase/lib/join.cljc
index 89a55149b51..12afd5cc5f4 100644
--- a/src/metabase/lib/join.cljc
+++ b/src/metabase/lib/join.cljc
@@ -1102,14 +1102,12 @@
(defn- join-lhs-display-name-for-first-join-in-first-stage
[query stage-number join-or-joinable]
- (when (and (zero? (lib.util/canonical-stage-index query stage-number)) ; first stage?
- (first-join? query stage-number join-or-joinable) ; first join?
- (lib.util/source-table-id query)) ; query ultimately uses source Table?
- (let [table-id (lib.util/source-table-id query)
- table (lib.metadata/table query table-id)]
+ (when-let [table (and (zero? (lib.util/canonical-stage-index query stage-number)) ; first stage?
+ (first-join? query stage-number join-or-joinable) ; first join?
+ (lib.metadata.calculation/primary-source-table query))] ; query ultimately uses source Table?
;; I think `:default` display name style is okay here, there shouldn't be a difference between `:default` and
;; `:long` for a Table anyway
- (lib.metadata.calculation/display-name query stage-number table))))
+ (lib.metadata.calculation/display-name query stage-number table)))
(mu/defn join-lhs-display-name :- ::lib.schema.common/non-blank-string
"Get the display name for whatever we are joining. See #32015 and #32764 for screenshot examples.
diff --git a/src/metabase/lib/metadata/calculation.cljc b/src/metabase/lib/metadata/calculation.cljc
index d28bd31c29b..65cc26024a8 100644
--- a/src/metabase/lib/metadata/calculation.cljc
+++ b/src/metabase/lib/metadata/calculation.cljc
@@ -734,3 +734,26 @@
(into [] (remove (comp #{:source/joins :source/implicitly-joinable}
:lib/source))
(returned-columns no-fields stage-number))))
+
+(mu/defn primary-source-table :- [:maybe ::lib.schema.metadata/table]
+ "If this query has an MBQL first stage with a `:source-table` ID, return the `:metadata/table` for it.
+
+ Returns nil if the query is native or has a `:source-card`."
+ [query]
+ (some->> query lib.util/source-table-id (lib.metadata/table query)))
+
+(mu/defn primary-source-card :- [:maybe ::lib.schema.metadata/card]
+ "If this query has an MBQL first stage with a `:source-card` ID, return the `:metadata/card` for it.
+
+ Returns nil if the query is native or has a `:source-table`."
+ [query]
+ (some->> query lib.util/source-card-id (lib.metadata/card query)))
+
+(mu/defn primary-source :- [:maybe [:or ::lib.schema.metadata/card ::lib.schema.metadata/table]]
+ "If this query has an MBQL first stage with a `:source-table` or `:source-card`, return the corresponding
+ `:metadata/table` or `:metadata/card`.
+
+ Returns nil if the query is native."
+ [query]
+ (or (primary-source-table query)
+ (primary-source-card query)))
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index d5f39736226..8894b3e13d3 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -201,7 +201,7 @@
(mu/defn- source-card-id :- [:maybe ::lib.schema.id/card]
[query :- [:maybe ::queries.schema/query]]
- (some-> query not-empty lib/source-card-id))
+ (some-> query not-empty lib/primary-source-card-id))
(mu/defn- card->integer-table-ids :- [:maybe [:set {:min 1} ::lib.schema.id/table]]
"Return integer source table ids for card's :dataset_query."
diff --git a/src/metabase/query_permissions/impl.clj b/src/metabase/query_permissions/impl.clj
index 1ee49aeb564..e1780251a7b 100644
--- a/src/metabase/query_permissions/impl.clj
+++ b/src/metabase/query_permissions/impl.clj
@@ -230,7 +230,7 @@
(if-let [source-card-id (some-> query
not-empty
(->> (lib-be/normalize-query metadata-provider))
- lib/source-card-id)]
+ lib/primary-source-card-id)]
{:paths (source-card-read-perms source-card-id)}
;; otherwise if there's no source card then calculate perms based on the Tables referenced in the query
(let [query (cond-> query
diff --git a/src/metabase/query_processor/api.clj b/src/metabase/query_processor/api.clj
index fb3f0ee8d60..2712a6029a3 100644
--- a/src/metabase/query_processor/api.clj
+++ b/src/metabase/query_processor/api.clj
@@ -46,7 +46,7 @@
well."
[query]
(when-let [source-card-id (and ((complement #{:internal "internal"}) (:type query))
- (some-> query not-empty lib-be/normalize-query lib/source-card-id))]
+ (some-> query not-empty lib-be/normalize-query lib/primary-source-card-id))]
(log/infof "Source query for this query is Card %s" (pr-str source-card-id))
(api/read-check :model/Card source-card-id)
source-card-id))
diff --git a/src/metabase/upload/impl.clj b/src/metabase/upload/impl.clj
index 417395e3179..b3df744d201 100644
--- a/src/metabase/upload/impl.clj
+++ b/src/metabase/upload/impl.clj
@@ -742,7 +742,7 @@
; dataset_query can be empty in tests
(when-let [query (not-empty query)]
(when (and (mbql? model) (no-joins? query))
- (lib/source-table-id query))))
+ (lib/primary-source-table-id query))))
(defn- invalidate-cached-models!
"Invalidate the model cache and result metadata for all models where `:based_on_upload` resolves to the given table."
diff --git a/src/metabase/xrays/automagic_dashboards/core.clj b/src/metabase/xrays/automagic_dashboards/core.clj
index c3d6ea470d8..9cf8b9351ba 100644
--- a/src/metabase/xrays/automagic_dashboards/core.clj
+++ b/src/metabase/xrays/automagic_dashboards/core.clj
@@ -239,7 +239,7 @@
(mu/defn- source-card-id [card-or-question :- [:map
[:dataset_query ::ads/query]]]
- (lib/source-card-id (:dataset_query card-or-question)))
+ (lib/primary-source-card-id (:dataset_query card-or-question)))
(mu/defn- nested-query?
"Is this card or question derived from another model or question?"
diff --git a/test/metabase/lib/metadata/calculation_test.cljc b/test/metabase/lib/metadata/calculation_test.cljc
index 4459f88bc50..b3081781f3d 100644
--- a/test/metabase/lib/metadata/calculation_test.cljc
+++ b/test/metabase/lib/metadata/calculation_test.cljc
@@ -1179,3 +1179,32 @@
(testing "retired column is NOT included even with :include-sensitive-fields? true"
(let [visible-col-ids (into #{} (map :id) (lib/visible-columns query -1 {:include-sensitive-fields? true}))]
(is (not (contains? visible-col-ids (meta/id :venues :longitude)))))))))
+
+(deftest ^:parallel primary-source-table-test
+ (testing `lib.metadata.calculation/primary-source-table
+ (let [table-query (lib.tu/venues-query)
+ mp lib.tu/metadata-provider-with-card
+ card-query (lib/query mp (lib.metadata/card mp 1))]
+ (testing "returns the `:metadata/table` for a query based on a table"
+ (is (= (meta/table-metadata :venues)
+ (lib.metadata.calculation/primary-source-table table-query)))
+ (testing "likewise for primary-source"
+ (is (= (meta/table-metadata :venues)
+ (lib.metadata.calculation/primary-source table-query)))))
+ (testing "returns nil for a query based on a card"
+ (is (nil? (lib.metadata.calculation/primary-source-table card-query)))))))
+
+(deftest ^:parallel primary-source-card-test
+ (testing `lib.metadata.calculation/primary-source-card
+ (let [table-query (lib.tu/venues-query)
+ mp (lib.tu/metadata-provider-with-mock-cards)
+ card (lib.metadata/card mp 1)
+ card-query (lib/query mp card)]
+ (testing "returns the `:metadata/card` for a query based on a card"
+ (is (= card
+ (lib.metadata.calculation/primary-source-card card-query)))
+ (testing "likewise for primary-source"
+ (is (= card
+ (lib.metadata.calculation/primary-source card-query)))))
+ (testing "returns nil for a query based on a table"
+ (is (nil? (lib.metadata.calculation/primary-source-card table-query)))))))
diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.clj
index 69d0664f710..5b0a77ad3a4 100644
--- a/test/metabase/xrays/automagic_dashboards/core_test.clj
+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj
@@ -562,7 +562,7 @@
(is (= source-database-id query-db-id))
(is (= source-table-id magic-card-table-id))
(is (= source-card-id
- (lib/source-card-id magic-card-query)))
+ (lib/primary-source-card-id magic-card-query)))
(doseq [breakout (lib/breakouts magic-card-query)
field-id (lib/all-field-ids breakout)]
(is (contains? valid-source-ids field-id)))))