Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Don't offer drill-throughs when top-level column doesn't exist (#66907)

Timothy S. Dean1aaee4e32c7f
diff --git a/src/metabase/lib/drill_thru/pivot.cljc b/src/metabase/lib/drill_thru/pivot.cljc
index 18ac7fa20a1..906dc499305 100644
--- a/src/metabase/lib/drill_thru/pivot.cljc
+++ b/src/metabase/lib/drill_thru/pivot.cljc
@@ -123,25 +123,33 @@
See `:pivots` key, which holds a map `{t [breakouts...]}` where `t` is `:category`, `:location`, or `:time`.
If a key is missing, there are no breakouts of that kind."
- [query :- ::lib.schema/query
- _stage-number :- :int
+ [query :- ::lib.schema/query
+ _stage-number :- :int
{:keys [column dimensions value] :as context} :- ::lib.schema.drill-thru/context]
- (let [stage-number (lib.underlying/top-level-stage-number query)]
+ (let [stage-number (lib.underlying/top-level-stage-number query)
+ ;; Filter dimensions to only those that can be traced back to the top-level query. Dimensions that reference
+ ;; expressions defined in later stages (after aggregation) cannot be used for filtering, since those
+ ;; expressions don't exist in the top-level query. See issue #66715.
+ traceable-dimensions (lib.underlying/traceable-dimensions query dimensions)
+ ;; If dimensions were provided but none are traceable, we can't offer this drill - it would create
+ ;; invalid filters. Only proceed when there were no dimensions to begin with or some are traceable.
+ dimensions-are-untraceable? (and (seq dimensions) (not traceable-dimensions))]
(when (and (lib.drill-thru.common/mbql-stage? query stage-number)
column
(some? value)
+ (not dimensions-are-untraceable?)
(lib.underlying/aggregation-sourced? query column)
(-> (lib.aggregation/aggregations query stage-number) count pos?))
(let [breakout-pivot-types (permitted-pivot-types query stage-number)
pivots (into {} (for [pivot-type breakout-pivot-types
- :let [pred (get pivot-type-predicates pivot-type)
- columns (pivot-drill-pred query stage-number context pred)]
- :when (not-empty columns)]
+ :let [pred (get pivot-type-predicates pivot-type)
+ columns (pivot-drill-pred query stage-number context pred)]
+ :when (not-empty columns)]
[pivot-type columns]))]
(when-not (empty? pivots)
{:lib/type :metabase.lib.drill-thru/drill-thru
:type :drill-thru/pivot
- :dimensions dimensions
+ :dimensions traceable-dimensions
:pivots pivots
:stage-number stage-number})))))
diff --git a/src/metabase/lib/drill_thru/underlying_records.cljc b/src/metabase/lib/drill_thru/underlying_records.cljc
index 61a232edded..a9bb1a1418f 100644
--- a/src/metabase/lib/drill_thru/underlying_records.cljc
+++ b/src/metabase/lib/drill_thru/underlying_records.cljc
@@ -28,7 +28,7 @@
Question transformation:
- Set display \"table\""
- (:refer-clojure :exclude [mapv empty? not-empty #?(:clj for)])
+ (:refer-clojure :exclude [mapv empty? #?(:clj for)])
(:require
[medley.core :as m]
[metabase.lib.aggregation :as lib.aggregation]
@@ -50,7 +50,7 @@
[metabase.lib.underlying :as lib.underlying]
[metabase.lib.util :as lib.util]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [mapv empty? not-empty #?(:clj for)]]))
+ [metabase.util.performance :refer [mapv empty? #?(:clj for)]]))
(mu/defn underlying-records-drill :- [:maybe ::lib.schema.drill-thru/drill-thru.underlying-records]
"When clicking on a particular broken-out group, offer a look at the details of all the rows that went into this
@@ -85,34 +85,43 @@
;; - dimensions holds only the legend's column, eg. Products.CATEGORY.
;; This function returns the table name and row count, since that's used for pluralization of the name.
- (when (and (lib.drill-thru.common/mbql-stage? query stage-number)
- (lib.underlying/has-aggregation-or-breakout? query)
- ;; Either we clicked the aggregation, or there are dimensions.
- (or (lib.underlying/aggregation-sourced? query column)
- (not-empty dimensions))
- ;; Either we need both column and value (cell/map/data point click) or neither (chart legend click).
- (or (and column (some? value))
- (and (nil? column) (nil? value)))
- ;; If the column exists, it must not be a structured column like JSON.
- (not (and column (lib.types.isa/structured? column))))
- {:lib/type :metabase.lib.drill-thru/drill-thru
- :type :drill-thru/underlying-records
- ;; TODO: This is a bit confused for non-COUNT aggregations. Perhaps it should just always be 10 or something?
- ;; Note that some languages have different plurals for exactly 2, or for 1, 2-5, and 6+.
- :row-count (if (and (number? value)
- (not (neg? value)))
- value
- 2)
- :table-name (when-let [table-or-card (or (some->> query lib.util/source-table-id (lib.metadata/table query))
- (some->> query lib.util/source-card-id (lib.metadata/card query)))]
- (lib.metadata.calculation/display-name query stage-number table-or-card))
- :dimensions dimensions
- ;; If the underlying column comes from an aggregation, then the column-ref needs to be updated as well to the
- ;; corresponding aggregation ref so that [[drill-underlying-records]] knows to extract the filter implied by
- ;; aggregations like sum-where.
- :column-ref (if (lib.underlying/strictly-underlying-aggregation? query column)
- (lib.aggregation/column-metadata->aggregation-ref (lib.underlying/top-level-column query column))
- column-ref)}))
+ ;; Filter dimensions to only those that can be traced back to the top-level query. Dimensions that reference
+ ;; expressions defined in later stages (after aggregation) cannot be used for filtering, since those expressions
+ ;; don't exist in the top-level query. See issue #66715.
+ (let [traceable-dimensions (lib.underlying/traceable-dimensions query dimensions)
+ ;; If dimensions were provided but none are traceable, we can't offer this drill - it would show
+ ;; unfiltered results which is confusing. Only allow aggregation-sourced clicks when there were
+ ;; no dimensions to begin with (single-row aggregation with no breakouts).
+ dimensions-are-untraceable? (and (seq dimensions) (not traceable-dimensions))]
+ (when (and (lib.drill-thru.common/mbql-stage? query stage-number)
+ (lib.underlying/has-aggregation-or-breakout? query)
+ (not dimensions-are-untraceable?)
+ ;; Either we clicked the aggregation, or there are traceable dimensions.
+ (or (lib.underlying/aggregation-sourced? query column)
+ traceable-dimensions)
+ ;; Either we need both column and value (cell/map/data point click) or neither (chart legend click).
+ (or (and column (some? value))
+ (and (nil? column) (nil? value)))
+ ;; If the column exists, it must not be a structured column like JSON.
+ (not (and column (lib.types.isa/structured? column))))
+ {:lib/type :metabase.lib.drill-thru/drill-thru
+ :type :drill-thru/underlying-records
+ ;; TODO: This is a bit confused for non-COUNT aggregations. Perhaps it should just always be 10 or something?
+ ;; Note that some languages have different plurals for exactly 2, or for 1, 2-5, and 6+.
+ :row-count (if (and (number? value)
+ (not (neg? value)))
+ value
+ 2)
+ :table-name (when-let [table-or-card (or (some->> query lib.util/source-table-id (lib.metadata/table query))
+ (some->> query lib.util/source-card-id (lib.metadata/card query)))]
+ (lib.metadata.calculation/display-name query stage-number table-or-card))
+ :dimensions traceable-dimensions
+ ;; If the underlying column comes from an aggregation, then the column-ref needs to be updated as well to the
+ ;; corresponding aggregation ref so that [[drill-underlying-records]] knows to extract the filter implied by
+ ;; aggregations like sum-where.
+ :column-ref (if (lib.underlying/strictly-underlying-aggregation? query column)
+ (lib.aggregation/column-metadata->aggregation-ref (lib.underlying/top-level-column query column))
+ column-ref)})))
(defmethod lib.drill-thru.common/drill-thru-info-method :drill-thru/underlying-records
[_query _stage-number {:keys [row-count table-name]}]
diff --git a/src/metabase/lib/underlying.cljc b/src/metabase/lib/underlying.cljc
index 6b8bfa4f0b1..039746e6825 100644
--- a/src/metabase/lib/underlying.cljc
+++ b/src/metabase/lib/underlying.cljc
@@ -1,7 +1,7 @@
(ns metabase.lib.underlying
"Helpers for getting at \"underlying\" or \"top-level\" queries and columns.
This logic is shared by a handful of things like drill-thrus."
- (:refer-clojure :exclude [empty? #?(:clj for)])
+ (:refer-clojure :exclude [empty? not-empty #?(:clj for)])
(:require
[clojure.set :as set]
[metabase.lib.aggregation :as lib.aggregation]
@@ -16,7 +16,7 @@
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.util :as lib.util]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [empty? #?(:clj for)]]))
+ [metabase.util.performance :refer [empty? not-empty #?(:clj for)]]))
(mu/defn- pop-until-aggregation-or-breakout :- [:tuple [:maybe ::lib.schema/query] [:int {:max -1}]]
"Strips off any trailing stages that do not contain aggregations or breakouts.
@@ -138,3 +138,17 @@
column :- [:maybe ::lib.schema.metadata/column]]
(and (not (aggregation-sourced? column))
(aggregation-sourced? query column)))
+
+(mu/defn traceable-dimensions :- [:maybe [:sequential :map]]
+ "Filter dimensions to only those whose columns can be traced back to the top-level query.
+
+ Returns nil if the result would be empty. This is used by drill-thrus to ensure they don't try to create filters
+ on columns (like expressions defined in later stages) that don't exist in the top-level query.
+
+ See issue #66715."
+ [query :- ::lib.schema/query
+ dimensions :- [:maybe [:sequential :map]]]
+ (not-empty
+ (for [dim dimensions
+ :when (top-level-column query (:column dim))]
+ dim)))
diff --git a/test/metabase/lib/drill_thru/pivot_test.cljc b/test/metabase/lib/drill_thru/pivot_test.cljc
index 33c673565cc..612f31c4c47 100644
--- a/test/metabase/lib/drill_thru/pivot_test.cljc
+++ b/test/metabase/lib/drill_thru/pivot_test.cljc
@@ -201,8 +201,8 @@
(defn- expecting [dim-names pivot-types]
{:expected {:type :drill-thru/pivot
- :dimensions (for [dim dim-names]
- {:column {:name dim}})
+ :dimensions (not-empty (for [dim dim-names]
+ {:column {:name dim}}))
:pivots (->> (for [typ [:category :location :time]]
[typ (if ((set pivot-types) typ)
not-empty
@@ -316,3 +316,36 @@
:custom-row {"IS_OPEN" true
"count" 10}}
(expecting ["IS_OPEN"] [:category :time])))))
+
+(deftest ^:parallel expression-after-aggregation-test
+ (testing "custom column defined after aggregation should not offer pivot drill (#66715)"
+ (let [;; Stage 0: Count of products by category
+ ;; Stage 1: Add custom column "Custom Category" that references the breakout result
+ base-query (-> (lib/query meta/metadata-provider (meta/table-metadata :products))
+ (lib/aggregate (lib/count))
+ (lib/breakout (meta/field-metadata :products :category))
+ lib/append-stage)
+ category-col (some #(when (= (:name %) "CATEGORY") %)
+ (lib/returned-columns base-query))
+ query (lib/expression base-query "Custom Category" (lib/ref category-col))
+ cols (lib/returned-columns query)
+ custom-cat-col (some #(when (= (:name %) "Custom Category") %) cols)
+ count-col (some #(when (= (:name %) "count") %) cols)
+ ;; Simulate clicking on a cell with Custom Category as the dimension
+ context {:column count-col
+ :column-ref (lib/ref count-col)
+ :value 42
+ :row [{:column custom-cat-col
+ :column-ref (lib/ref custom-cat-col)
+ :value "Doohickey"}
+ {:column count-col
+ :column-ref (lib/ref count-col)
+ :value 42}]
+ :dimensions [{:column custom-cat-col
+ :column-ref (lib/ref custom-cat-col)
+ :value "Doohickey"}]}
+ ;; No pivot drill:
+ drill (some #(when (= (:type %) :drill-thru/pivot) %)
+ (lib/available-drill-thrus query context))]
+ (testing "drill should not be available when dimension can't be traced back"
+ (is (nil? drill))))))
diff --git a/test/metabase/lib/drill_thru/underlying_records_test.cljc b/test/metabase/lib/drill_thru/underlying_records_test.cljc
index 9151dc8e48f..2f288d176ea 100644
--- a/test/metabase/lib/drill_thru/underlying_records_test.cljc
+++ b/test/metabase/lib/drill_thru/underlying_records_test.cljc
@@ -605,6 +605,41 @@
(is (=? [[:= {} [:expression {} "cost bucket"] "12"]]
(lib/filters (lib/drill-thru query drill)))))))
+(deftest ^:parallel expression-after-aggregation-test
+ (testing "custom column defined after aggregation should not offer underlying-records drill (#66715)"
+ (let [;; Stage 0: Count of products by category
+ ;; Stage 1: Add custom column "Custom Category" that references the breakout result
+ base-query (-> (lib/query meta/metadata-provider (meta/table-metadata :products))
+ (lib/aggregate (lib/count))
+ (lib/breakout (meta/field-metadata :products :category))
+ lib/append-stage)
+ category-col (m/find-first #(= (:name %) "CATEGORY")
+ (lib/returned-columns base-query))
+ query (lib/expression base-query "Custom Category" (lib/ref category-col))
+ cols (lib/returned-columns query)
+ custom-cat-col (m/find-first #(= (:name %) "Custom Category") cols)
+ count-col (m/find-first #(= (:name %) "count") cols)
+ ;; Simulate clicking on a bar chart with Custom Category on X axis
+ context {:column custom-cat-col
+ :column-ref (lib/ref custom-cat-col)
+ :value 42
+ :row [{:column custom-cat-col
+ :column-ref (lib/ref custom-cat-col)
+ :value "Doohickey"}
+ {:column count-col
+ :column-ref (lib/ref count-col)
+ :value 42}]
+ :dimensions [{:column custom-cat-col
+ :column-ref (lib/ref custom-cat-col)
+ :value "Doohickey"}]}
+ ;; No underlying-records drill:
+ drill (m/find-first #(= (:type %) :drill-thru/underlying-records)
+ (lib/available-drill-thrus query context))]
+ (testing "Custom Category expression can't be traced to top-level query"
+ (is (nil? (lib.underlying/top-level-column query custom-cat-col))))
+ (testing "drill should not be available when dimension can't be traced back"
+ (is (nil? drill))))))
+
(deftest ^:parallel native-query-with-multiple-breakouts-on-same-column-test
(let [mp (lib.tu/mock-metadata-provider
meta/metadata-provider