Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Compute param values from card results (#64318)

lbrdnk2693173e7f58
diff --git a/e2e/support/helpers/e2e-downloads-helpers.ts b/e2e/support/helpers/e2e-downloads-helpers.ts
index 83d0519a15a..8f143c6e175 100644
--- a/e2e/support/helpers/e2e-downloads-helpers.ts
+++ b/e2e/support/helpers/e2e-downloads-helpers.ts
@@ -94,6 +94,10 @@ export function downloadAndAssert(
cy.log(`Downloading ${fileType} file`);
+ // For _some_ reason wait is necessary for dashcard menu to stay open after
+ // the click. I assume card needs to finish loading.
+ cy.wait(200);
+
if (isDashboard) {
if (isEmbed) {
getDashboardCard().realHover();
diff --git a/e2e/test/scenarios/admin/i18n/content-translation/dashboards.cy.spec.ts b/e2e/test/scenarios/admin/i18n/content-translation/dashboards.cy.spec.ts
index d694f816381..a4fd1eb89ea 100644
--- a/e2e/test/scenarios/admin/i18n/content-translation/dashboards.cy.spec.ts
+++ b/e2e/test/scenarios/admin/i18n/content-translation/dashboards.cy.spec.ts
@@ -768,7 +768,6 @@ describe("scenarios > content translation > static embedding > dashboards", () =
H.filterWidget().findByText("Multi").click();
// Search matches against untranslated text, hence "Fran" matching these names
cy.findByPlaceholderText("Recherche dans la liste").type("Fran");
- cy.wait("@searchQuery");
cy.findByTestId("parameter-value-dropdown").within(() => {
cy.findByText(/Glacia Froskeon/).click();
cy.button(/Ajouter un filtre/).click();
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index 6ed73bf54bf..6007b3490e8 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -472,8 +472,10 @@
temporal-bucket
with-temporal-bucket]
[lib.util
+ canonical-stage-index
clause?
clause-of-type?
+ drop-summary-clauses
fresh-uuids
mbql-stage?
native-stage?
diff --git a/src/metabase/lib/equality.cljc b/src/metabase/lib/equality.cljc
index d11f08ccfd2..2cbd20a32f7 100644
--- a/src/metabase/lib/equality.cljc
+++ b/src/metabase/lib/equality.cljc
@@ -201,7 +201,7 @@
source-field-join-alias]} _ref-id] :- ::lib.schema.ref/ref
column :- ::lib.schema.metadata/column]
(if source-field
- (and (clojure.core/= source-field (:fk-field-id column))
+ (and (clojure.core/= source-field ((some-fn :fk-field-id :lib/original-fk-field-id) column))
;; `source-field-name` is not available on old refs
(or (nil? source-field-name) (clojure.core/= source-field-name (:fk-field-name column)))
(clojure.core/= source-field-join-alias (:fk-join-alias column)))
diff --git a/src/metabase/lib/stage.cljc b/src/metabase/lib/stage.cljc
index f2a4b9a6201..48c77f25cef 100644
--- a/src/metabase/lib/stage.cljc
+++ b/src/metabase/lib/stage.cljc
@@ -60,7 +60,10 @@
:mbql.stage/mbql :source/card)]
(not-empty
(into []
- (comp (map #(assoc % :lib/source source-type))
+ (comp (map (fn [col]
+ (cond-> (assoc col :lib/source source-type)
+ (= :source/card source-type) (-> (assoc :lib/card-id source-card)
+ (dissoc :lib/expression-name)))))
;; do not truncate the desired column aliases coming back from a native query, because if a
;; native query returns a 'crazy long' column name then we need to use that in the next stage.
;; See [[metabase.lib.stage-test/propagate-crazy-long-native-identifiers-test]]
diff --git a/src/metabase/lib/util.cljc b/src/metabase/lib/util.cljc
index 323f8f4de3d..a3a6670c77f 100644
--- a/src/metabase/lib/util.cljc
+++ b/src/metabase/lib/util.cljc
@@ -529,3 +529,10 @@
(:query :native) :mbql-version/legacy
;; otherwise, this is not a valid MBQL query.
nil))
+
+(defn drop-summary-clauses
+ "Remove :aggregation and :breakout from the stage at `stage-number` of a `query`."
+ ([query]
+ (drop-summary-clauses query -1))
+ ([query stage-number]
+ (update-query-stage query stage-number dissoc :aggregation :breakout)))
diff --git a/src/metabase/parameters/custom_values.clj b/src/metabase/parameters/custom_values.clj
index 157ede25675..d0330f685d8 100644
--- a/src/metabase/parameters/custom_values.clj
+++ b/src/metabase/parameters/custom_values.clj
@@ -19,7 +19,6 @@
[metabase.query-processor.util :as qp.util]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
- [metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
@@ -62,6 +61,44 @@
Maybe we should lower it for the sake of displaying a parameter dropdown."
1000)
+(defn- query-and-value-column
+ "Searches stages of card's dataset query for column matching value-field-ref. Searching from -1 to 0th stage,
+ removing the later stages. Returns map as
+ {:query ...
+ :value-column ...}.
+ Query contains stages where last one is empty, having the matching column available.
+ Value column contains that column.
+ Callers can use column to build something beautiful on the last empty stage (see [[values-from-card-query]]).
+ Returns nil if column is not found."
+ [card value-field-ref]
+ (when-some [mp (-> card :dataset_query :lib/metadata)]
+ (let [card (lib.metadata/card mp (:id card))
+ full-query (lib/card->underlying-query mp card)]
+ (loop [stage-number (lib/canonical-stage-index full-query -1)
+ query full-query]
+ ;; (1) Append stage. The searched column may be part of summaries. Further transformations will add
+ ;; filters and breakouts. A new stage ensures those operations are not conflated with existing summaries.
+ ;; (2) visible-columns. Visible columns are used because ref may come from implicit join.
+ (let [query (lib/append-stage query)]
+ (or
+ ;; Examine the query with summaries first.
+ (when-some [value-column (lib/find-matching-column
+ query -1
+ value-field-ref (lib/visible-columns query))]
+ {:query query
+ :value-column value-column})
+ ;; If column was not found, remove the summaries.
+ (let [query (lib/drop-summary-clauses query stage-number)]
+ (when-some [value-column (lib/find-matching-column
+ query -1
+ value-field-ref (lib/visible-columns query))]
+ {:query query
+ :value-column value-column}))
+ ;; If not found, drop the newly added stage and examined stage, then continue searching.
+ (when (pos-int? stage-number)
+ (recur (-> query lib/drop-stage lib/drop-stage)
+ (dec stage-number)))))))))
+
(mr/def ::values-from-card-query.options
[:map
;; despite this being called "query string" it can actually be any value because it just gets used in an `:=`
@@ -69,37 +106,29 @@
[:query-string {:optional true} :any]])
(mu/defn- values-from-card-query :- [:maybe ::lib.schema/query]
- [{query :dataset_query, :keys [id], :as _card} :- [:and
- :metabase.queries.schema/card
- [:map
- [:id ::lib.schema.id/card]]]
+ [{query :dataset_query, :as card} :- [:and
+ :metabase.queries.schema/card
+ [:map
+ [:id ::lib.schema.id/card]]]
field-ref :- [:or :mbql.clause/field :mbql.clause/expression]
{:keys [query-string] :as _opts} :- [:maybe ::values-from-card-query.options]]
(when (seq query)
- ;; start a new query using this Card as a starting point
- (let [query (lib/query query (lib.metadata/card query id))]
- (when-let [visible-columns (or (not-empty (lib/visible-columns query))
- (log/warnf "Cannot get values from Card %d: Card query has no visible columns"
- id))]
- (when-let [value-column (or (lib/find-matching-column query -1 field-ref visible-columns)
- (log/warnf "Cannot get values from Card %d: failed to find column for ref %s\nFound: %s"
- id
- (pr-str field-ref)
- (pr-str (map (some-fn :lib/source-column-alias :name) visible-columns))))]
- (let [textual? (lib.types.isa/string? value-column)
- nonempty ((if textual? lib/not-empty lib/not-null) value-column)
- query-filter (when query-string
- (if textual?
- (lib/contains (lib/lower value-column) (u/lower-case-en query-string))
- (lib/= value-column query-string)))]
- (-> query
- (lib/limit *max-rows*)
- (lib/filter nonempty)
- (cond-> #_query query-filter (lib/filter query-filter))
- (lib/breakout value-column)
- ;; TODO(Braden, 07/04/2025): This should probably become a lib helper? I suspect this isn't the only
- ;; "internal" query in the BE.
- (assoc-in [:middleware :disable-remaps?] true))))))))
+ (let [{:keys [query value-column]} (query-and-value-column card field-ref)]
+ (when value-column
+ (let [textual? (lib.types.isa/string? value-column)
+ nonempty ((if textual? lib/not-empty lib/not-null) value-column)
+ query-filter (when query-string
+ (if textual?
+ (lib/contains (lib/lower value-column) (u/lower-case-en query-string))
+ (lib/= value-column query-string)))]
+ (-> query
+ (lib/limit *max-rows*)
+ (lib/filter nonempty)
+ (cond-> #_query query-filter (lib/filter query-filter))
+ (lib/breakout value-column)
+ ;; TODO(Braden, 07/04/2025): This should probably become a lib helper? I suspect this isn't the only
+ ;; "internal" query in the BE.
+ (assoc-in [:middleware :disable-remaps?] true)))))))
(mu/defn values-from-card
"Get distinct values of a field from a card.
diff --git a/src/metabase/parameters/dashboard.clj b/src/metabase/parameters/dashboard.clj
index d64c0e0d381..361d328f234 100644
--- a/src/metabase/parameters/dashboard.clj
+++ b/src/metabase/parameters/dashboard.clj
@@ -2,7 +2,10 @@
(:require
[medley.core :as m]
[metabase.api.common :as api]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.schema.parameter :as lib.schema.parameter]
[metabase.parameters.chain-filter :as chain-filter]
[metabase.parameters.custom-values :as custom-values]
@@ -83,6 +86,26 @@
2 second
1 first)))))
+(mu/defn- cards-with-filters :- [:sequential ::lib.schema.metadata/card]
+ "Lazy seq of cards from `dashboard` that are mapped to param `param-key` and contain some filters.
+ Dashboard is expected to have hydrated `:resolved-params`."
+ [dashboard param-key]
+ (for [mapping (get-in dashboard [:resolved-params param-key :mappings])
+ :let [database-id (get-in mapping [:dashcard :card :dataset_query :database])
+ card-id (get-in mapping [:dashcard :card :id])
+ mp (lib-be/application-database-metadata-provider database-id)
+ card (lib.metadata/card mp card-id)
+ query (lib/card->underlying-query mp card)]
+ :when (and (not-empty query)
+ (m/find-first (partial lib/filters query)
+ (range (lib/stage-count query))))]
+ card))
+
+(defn- cards-with-filters?
+ "Does param-key have any card with filter mapped? Dashboard is expected to have hydrated `:resolved-params`."
+ [dashboard param-key]
+ (boolean (seq (cards-with-filters dashboard param-key))))
+
(mu/defn chain-filter :- ms/FieldValuesResult
"C H A I N filters!
@@ -99,7 +122,8 @@
constraints (chain-filter-constraints dashboard constraint-param-key->value)
param (get-in dashboard [:resolved-params param-key])
field-ids (into #{} (map :field-id (param->fields param)))]
- (if (empty? field-ids)
+ (if (or (empty? field-ids)
+ (cards-with-filters? dashboard param-key))
(or (filter-values-from-field-refs dashboard param-key)
(throw (ex-info (tru "Parameter {0} does not have any Fields associated with it" (pr-str param-key))
{:param (get (:resolved-params dashboard) param-key)
diff --git a/src/metabase/query_processor/middleware/parameters/mbql.clj b/src/metabase/query_processor/middleware/parameters/mbql.clj
index 075907a677d..b486f615a48 100644
--- a/src/metabase/query_processor/middleware/parameters/mbql.clj
+++ b/src/metabase/query_processor/middleware/parameters/mbql.clj
@@ -112,7 +112,7 @@
(mu/defn- update-breakout-unit* :- ::lib.schema/stage
[stage :- ::lib.schema/stage
target-column :- [:or ::lib.schema.id/field :string]
- temporal-unit :- ::lib.schema.temporal-bucketing/unit
+ temporal-unit :- [:maybe ::lib.schema.temporal-bucketing/unit]
new-unit :- ::lib.schema.temporal-bucketing/unit]
(lib.util.match/replace stage
[(tag :guard #{:field :expression})
diff --git a/test/metabase/dashboards/api_test.clj b/test/metabase/dashboards/api_test.clj
index db85cdfd5e9..1e9995348ea 100644
--- a/test/metabase/dashboards/api_test.clj
+++ b/test/metabase/dashboards/api_test.clj
@@ -5423,3 +5423,114 @@
})]
(is (partial= {:name "Test Dashboard"
:creator_id (mt/user->id :crowberto)} response))))
+
+(deftest param-mapped-SAME-fields-filtered-cards-test
+ (testing "Values for param mapped to _same_ field in _different_ cards containing filters are computed correctly (#41684)"
+ (let [mp (mt/metadata-provider)
+ param-key "p"
+ base-queries [(-> (lib/query mp (lib.metadata/table mp (mt/id :categories)))
+ (lib/filter (lib/in (lib.metadata/field mp (mt/id :categories :name))
+ "African" "Artisan" "Bakery")))
+
+ (-> (lib/query mp (lib.metadata/table mp (mt/id :categories)))
+ (lib/filter (lib/in (lib.metadata/field mp (mt/id :categories :name))
+ "African" "Artisan")))]
+ tests [{:test-case "single level queries"
+ :queries base-queries
+ :expectations [["African"] ["Artisan"] ["Bakery"]]}
+
+ {:test-case "nested filters"
+ :queries (mapv (fn [query]
+ ;; dummy stage
+ (-> (lib/append-stage query)
+ (lib/expression "e" (lib/concat (lib.metadata/field mp (mt/id :categories :name))
+ "XXX"))))
+ base-queries)
+ :expectations [["African"] ["Artisan"] ["Bakery"]]}]]
+ (doseq [{:keys [test-case queries expectations]} tests]
+ (mt/with-temp
+ [:model/Card
+ c1
+ {:dataset_query (queries 0)}
+
+ :model/Card
+ c2
+ {:dataset_query (queries 1)}
+
+ :model/Dashboard
+ d
+ {:parameters [{:id param-key
+ :name "filtered names filter"
+ :slug param-key
+ :type "string/="
+ :sectionId "string"}]}
+
+ :model/DashboardCard
+ _
+ {:dashboard_id (:id d)
+ :card_id (:id c1)
+ :parameter_mappings [{:card_id (:id c1)
+ :parameter_id param-key
+ :target ["dimension" ["field" (mt/id :categories :name) nil]]}]}
+
+ :model/DashboardCard
+ _
+ {:dashboard_id (:id d)
+ :card_id (:id c2)
+ :parameter_mappings [{:card_id (:id c2)
+ :parameter_id param-key
+ :target ["dimension" ["field" (mt/id :categories :name) nil]]}]}]
+
+ (testing test-case
+ (is (=? {:values expectations}
+ (mt/user-http-request :crowberto :get 200 (format "dashboard/%d/params/%s/values"
+ (:id d) param-key))))))))))
+
+(deftest param-mapped-DIFFERENT-fields-filtered-cards-test
+ (testing "Values for param mapped to different fields in cards containing filters are computed correctly (#41684)"
+ (let [mp (mt/metadata-provider)
+ param-key "p"]
+ (mt/with-temp
+ [:model/Card
+ c1
+ {:dataset_query (-> (lib/query mp (lib.metadata/table mp (mt/id :categories)))
+ (lib/filter (lib/in (lib.metadata/field mp (mt/id :categories :name))
+ "African" "Artisan" "Bakery")))}
+
+ :model/Card
+ c2
+ {:dataset_query (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))
+ (lib/filter (lib/in (lib.metadata/field mp (mt/id :venues :name))
+ "Red Medicine" "Krua Siri"))
+ (lib/append-stage)
+ (lib/expression "e" (lib/concat (lib.metadata/field mp (mt/id :categories :name))
+ "XXX")))}
+
+ :model/Dashboard
+ d
+ {:parameters [{:id param-key
+ :name "filtered names filter"
+ :slug param-key
+ :type "string/="
+ :sectionId "string"}]}
+
+ :model/DashboardCard
+ _
+ {:dashboard_id (:id d)
+ :card_id (:id c1)
+ :parameter_mappings [{:card_id (:id c1)
+ :parameter_id param-key
+ :target ["dimension" ["field" (mt/id :categories :name) nil]]}]}
+
+ :model/DashboardCard
+ _
+ {:dashboard_id (:id d)
+ :card_id (:id c2)
+ :parameter_mappings [{:card_id (:id c2)
+ :parameter_id param-key
+ :target ["dimension" ["field" (mt/id :venues :name) nil]]}]}]
+
+ (testing "Param mapped to different fields with different filters"
+ (is (=? {:values [["African"] ["Artisan"] ["Bakery"] ["Krua Siri"] ["Red Medicine"]]}
+ (mt/user-http-request :crowberto :get 200 (format "dashboard/%d/params/%s/values"
+ (:id d) param-key)))))))))