Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Limit body-only dashboard subscription results (#78431)

Ngoc Khuat7b60d66387af
diff --git a/e2e/test/scenarios/sharing/subscriptions.cy.spec.js b/e2e/test/scenarios/sharing/subscriptions.cy.spec.js
index 63651dd075e..4e84ca460b3 100644
--- a/e2e/test/scenarios/sharing/subscriptions.cy.spec.js
+++ b/e2e/test/scenarios/sharing/subscriptions.cy.spec.js
@@ -775,7 +775,11 @@ describe("scenarios > dashboard > subscriptions", () => {
cy.get(".container").within(() => {
cy.findByText("Total Orders");
- cy.findAllByText("18,760").should("have.length", 2);
+ // the scalar counts all orders, but the body-only Orders table is
+ // capped at the 2,000-row display limit so its truncation note no
+ // longer says 18,760 (GDGT-2773)
+ cy.findAllByText("18,760").should("have.length", 1);
+ cy.findByText("2,000");
});
});
diff --git a/src/metabase/notification/payload/execute.clj b/src/metabase/notification/payload/execute.clj
index defcdfee83c..517cc5aeecd 100644
--- a/src/metabase/notification/payload/execute.clj
+++ b/src/metabase/notification/payload/execute.clj
@@ -178,14 +178,19 @@
(defn execute-dashboard-subscription-card
"Returns subscription result for a card.
- `spill-budget` is the shared [[metabase.notification.payload.temp-storage/ResidentBudget]] for the whole
- notification, so memory held by sibling cards influences when this card spills to disk. When omitted (e.g. executing a
- card in isolation), a private budget is used so only the per-card spill limit applies.
+ Options:
+ - `:spill-budget` the shared [[metabase.notification.payload.temp-storage/ResidentBudget]] for the whole
+ notification, so memory held by sibling cards influences when this card spills to disk. When omitted (e.g.
+ executing a card in isolation), a private budget is used so only the per-card spill limit applies.
+ - `:attached?` whether this card's results are exported as a file attachment. Attached cards run to the
+ attachment row limit; body-only cards get the interactive display limits since only a preview is rendered.
This function should be executed under pulse's creator permissions."
([dashcard parameters]
- (execute-dashboard-subscription-card dashcard parameters (new-spill-budget)))
- ([{:keys [card_id dashboard_id] :as dashcard} parameters spill-budget]
+ (execute-dashboard-subscription-card dashcard parameters nil))
+ ([{:keys [card_id dashboard_id] :as dashcard} parameters
+ {:keys [spill-budget attached?]
+ :or {spill-budget (new-spill-budget)}}]
(log/with-context {:card_id card_id}
(try
(when-let [card (t2/select-one :model/Card :id card_id :archived false)]
@@ -194,7 +199,8 @@
result-fn (fn [card-id]
(let [card (if (= card-id (:id card))
card
- (t2/select-one :model/Card :id card-id))]
+ (t2/select-one :model/Card :id card-id))
+ attached-result? (and attached? (= card-id card_id))]
{:card card
:dashcard dashcard
;; TODO should this be dashcard?
@@ -209,7 +215,10 @@
:constraints {}
:middleware {:process-viz-settings? true
:js-int-to-string? false
- :add-default-userland-constraints? false}
+ ;; body-only cards only render a preview, so cap
+ ;; them at the interactive display limits;
+ ;; attachments need the full result
+ :add-default-userland-constraints? (not attached-result?)}
:make-run (fn make-run [qp _export-format]
(^:once fn* [query info]
(qp
@@ -238,18 +247,19 @@
(defn- dashcard->part
"Given a dashcard returns its part based on its type.
- `spill-budget` is the notification-wide [[metabase.notification.payload.temp-storage/ResidentBudget]] shared across
- all dashcards.
+ `opts` are the [[execute-dashboard]] options, shared across all dashcards.
The result will follow the pulse's creator permissions."
- [dashcard parameters spill-budget]
+ [dashcard parameters {:keys [spill-budget attached-card-ids]}]
(assert api/*current-user-id* "Makes sure you wrapped this with a `with-current-user`.")
(cond
(:card_id dashcard)
(log/with-context {:card_id (:card_id dashcard)}
(let [parameters (merge-default-values parameters)]
;; Streaming to disk is now handled by the query processor rff
- (-> (execute-dashboard-subscription-card dashcard parameters spill-budget)
+ (-> (execute-dashboard-subscription-card dashcard parameters
+ {:spill-budget spill-budget
+ :attached? (contains? attached-card-ids (:card_id dashcard))})
(m/update-existing :dashcard resolve-inline-parameters parameters))))
(virtual-card-of-type? dashcard "iframe")
@@ -282,12 +292,12 @@
(assoc :type :text)))))
(defn- dashcards->part
- "Render `dashcards` to parts, sharing `spill-budget` across them so memory held by earlier cards can push later (large)
- cards to spill to disk instead of collectively exhausting heap. The budget is created once per dashboard (spanning all
- tabs) by the caller, not here, so a multi-tab dashboard shares a single budget."
- [dashcards parameters spill-budget]
+ "Render `dashcards` to parts, sharing the `:spill-budget` in `opts` across them so memory held by earlier cards can
+ push later (large) cards to spill to disk instead of collectively exhausting heap. The budget is created once per
+ dashboard (spanning all tabs) by the caller, not here, so a multi-tab dashboard shares a single budget."
+ [dashcards parameters opts]
(let [ordered-dashcards (sort dashboard-card/dashcard-comparator dashcards)]
- (doall (keep #(dashcard->part % parameters spill-budget) ordered-dashcards))))
+ (doall (keep #(dashcard->part % parameters opts) ordered-dashcards))))
(mr/def ::Part
"Part."
@@ -313,13 +323,18 @@
(all tabs), so cards can't collectively exhaust heap regardless of how they're split into tabs. Defaults to the
production budget; pass your own (e.g. with lower limits) for testing.
- `:only-card-ids` when set, only dashcards whose :card_id is in this set are executed (e.g. attachment-only
- subscriptions that never render the other cards)."
+ subscriptions that never render the other cards).
+ - `:attached-card-ids` cards whose results are exported as file attachments; they run to the attachment row limit
+ while the rest get the interactive display limits."
([dashboard-id user-id parameters]
(execute-dashboard dashboard-id user-id parameters nil))
- ([dashboard-id user-id parameters {:keys [spill-budget only-card-ids] :or {spill-budget (new-spill-budget)}}]
- (let [keep-dashcards (fn [dashcards]
- (cond->> dashcards
- only-card-ids (filter #(contains? only-card-ids (:card_id %)))))]
+ ([dashboard-id user-id parameters {:keys [spill-budget only-card-ids attached-card-ids]
+ :or {spill-budget (new-spill-budget)}}]
+ (let [opts {:spill-budget spill-budget
+ :attached-card-ids attached-card-ids}
+ keep-dashcards (fn [dashcards]
+ (cond->> dashcards
+ only-card-ids (filter #(contains? only-card-ids (:card_id %)))))]
(request/with-current-user user-id
(if (render-tabs? dashboard-id)
(let [tabs (t2/hydrate (t2/select :model/DashboardTab :dashboard_id dashboard-id) :tab-cards)
@@ -333,10 +348,10 @@
(concat
(when should-render-tab?
[(tab->part tab)])
- (dashcards->part cards parameters spill-budget)))))))
+ (dashcards->part cards parameters opts)))))))
(let [dashcards (keep-dashcards (t2/select :model/DashboardCard :dashboard_id dashboard-id))]
(log/debugf "Rendering dashboard with %d cards" (count dashcards))
- (dashcards->part dashcards parameters spill-budget)))))))
+ (dashcards->part dashcards parameters opts)))))))
(mu/defn execute-card :- [:maybe ::Part]
"Returns the result for a card."
diff --git a/src/metabase/notification/payload/impl/dashboard.clj b/src/metabase/notification/payload/impl/dashboard.clj
index a7abcef9ee6..9d5a51a9201 100644
--- a/src/metabase/notification/payload/impl/dashboard.clj
+++ b/src/metabase/notification/payload/impl/dashboard.clj
@@ -45,10 +45,12 @@
(let [dashboard-id (:dashboard_id dashboard_subscription)
dashboard (t2/hydrate (t2/select-one :model/Dashboard dashboard-id) :tabs)
parameters (parameters (:parameters dashboard_subscription) (:parameters dashboard))
+ attached-ids (attached-card-ids dashboard_subscription)
only-card-ids (when (attachment-only? handlers)
- (attached-card-ids dashboard_subscription))]
+ attached-ids)]
{:dashboard_parts (cond->> (notification.execute/execute-dashboard dashboard-id creator_id parameters
- {:only-card-ids only-card-ids})
+ {:only-card-ids only-card-ids
+ :attached-card-ids attached-ids})
(:skip_if_empty dashboard_subscription)
(remove (fn [{part-type :type :as part}]
(and
diff --git a/test/metabase/notification/payload/execute_test.clj b/test/metabase/notification/payload/execute_test.clj
index e78d073aced..2e5a8791e69 100644
--- a/test/metabase/notification/payload/execute_test.clj
+++ b/test/metabase/notification/payload/execute_test.clj
@@ -57,6 +57,37 @@
(defn- spilled-part? [part]
(temp-storage/streaming-temp-file? (-> part :result :data :rows)))
+(deftest body-only-cards-use-display-limit-test
+ (testing "cards not selected for attachment get the interactive display limit; attached cards keep the attachment limit (GDGT-2773)"
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}
+ :model/Dashboard {dash-id :id} {}
+ :model/DashboardCard _ {:dashboard_id dash-id :card_id card-id}]
+ (let [row-count (fn [opts]
+ (let [part (first (card-parts (notification.payload.execute/execute-dashboard
+ dash-id (mt/user->id :rasta) [] opts)))]
+ (temp-storage/cleanup! (-> part :result :data :rows))
+ (-> part :result :row_count)))]
+ (is (= 2000 (row-count nil)))
+ (is (< 2000 (row-count {:attached-card-ids #{card-id}})))))))
+
+(deftest attached-card-series-uses-display-limit-test
+ (testing "additional series are display-only even when the primary card is attached"
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}
+ :model/Card {series-card-id :id} {:dataset_query (mt/mbql-query orders)}
+ :model/Dashboard {dash-id :id} {}
+ :model/DashboardCard {dashcard-id :id} {:dashboard_id dash-id :card_id card-id}
+ :model/DashboardCardSeries _ {:dashboardcard_id dashcard-id
+ :card_id series-card-id
+ :position 0}]
+ (let [part (first (card-parts (notification.payload.execute/execute-dashboard
+ dash-id (mt/user->id :rasta) []
+ {:attached-card-ids #{card-id}})))
+ series-result (-> part :dashcard :series-results first)]
+ (is (< 2000 (-> part :result :row_count)) "attached primary card keeps the attachment limit")
+ (is (= 2000 (-> series-result :result :row_count)) "display-only series gets the interactive limit")
+ (temp-storage/cleanup! (-> part :result :data :rows))
+ (temp-storage/cleanup! (-> series-result :result :data :rows))))))
+
(deftest tabbed-dashboard-shares-one-spill-budget-test
(testing "the resident-memory budget is shared across a multi-tab dashboard, so cards in different tabs collectively
count toward the spill cap (otherwise many small cards across tabs could exhaust memory)"
diff --git a/test/metabase/pulse/dashboard_subscription_test.clj b/test/metabase/pulse/dashboard_subscription_test.clj
index afb643b3774..6e2fa951f06 100644
--- a/test/metabase/pulse/dashboard_subscription_test.clj
+++ b/test/metabase/pulse/dashboard_subscription_test.clj
@@ -1496,9 +1496,9 @@
(let [executed (atom [])
orig (mt/original-fn #'notification.payload.execute/execute-dashboard-subscription-card)]
(mt/with-dynamic-fn-redefs [notification.payload.execute/execute-dashboard-subscription-card
- (fn [dashcard parameters spill-budget]
+ (fn [dashcard parameters opts]
(swap! executed conj (:card_id dashcard))
- (orig dashcard parameters spill-budget))]
+ (orig dashcard parameters opts))]
(f))
executed))
@@ -1549,6 +1549,39 @@
(pulse.send/send-pulse! (t2/select-one :model/Pulse pulse-id))))]
(is (= #{attached-id other-id} (set @executed))))))))
+(deftest dashboard-sub-body-only-cards-use-display-limit-test
+ (testing "cards not selected for attachment get the interactive display limit; attached cards keep the attachment limit (GDGT-2773)"
+ (mt/with-temp [:model/Card {attached-id :id} {:name "Attached Card"
+ :dataset_query (mt/mbql-query orders)}
+ :model/Card {body-only-id :id} {:name "Body Only Card"
+ :dataset_query (mt/mbql-query orders)}
+ :model/Dashboard {dashboard-id :id} {:name "Aviary KPIs"}
+ :model/DashboardCard _ {:dashboard_id dashboard-id :card_id attached-id}
+ :model/DashboardCard _ {:dashboard_id dashboard-id :card_id body-only-id}
+ :model/Pulse {pulse-id :id} {:name "Pulse Name"
+ :dashboard_id dashboard-id}
+ :model/PulseCard _ {:pulse_id pulse-id
+ :card_id attached-id
+ :position 0
+ :include_csv true}
+ :model/PulseCard _ {:pulse_id pulse-id
+ :card_id body-only-id
+ :position 1}
+ :model/PulseChannel {pc-id :id} {:pulse_id pulse-id
+ :channel_type "email"}
+ :model/PulseChannelRecipient _ {:user_id (pulse.test-util/rasta-id)
+ :pulse_channel_id pc-id}]
+ (let [row-counts (atom {})
+ orig (mt/original-fn #'notification.payload.execute/execute-dashboard-subscription-card)]
+ (mt/with-dynamic-fn-redefs [notification.payload.execute/execute-dashboard-subscription-card
+ (fn [dashcard parameters opts]
+ (u/prog1 (orig dashcard parameters opts)
+ (swap! row-counts assoc (:card_id dashcard) (-> <> :result :row_count))))]
+ (pulse.test-util/with-captured-channel-send-messages!
+ (pulse.send/send-pulse! (t2/select-one :model/Pulse pulse-id))))
+ (is (< 2000 (get @row-counts attached-id)) "attached card runs to the attachment limit")
+ (is (= 2000 (get @row-counts body-only-id)) "body-only card gets the interactive display limit")))))
+
(defn- pdf->text
"All text extracted from the rendered PDF `bytes`. Card titles/headings are drawn as native, selectable text, so a
card that was rendered leaves its title here and an omitted card leaves nothing."