Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[PDFs] Reuse query results and PNG rendering memory pools for PDFs (#76918)

Braden Shepherdsonb8d634be905d
diff --git a/src/metabase/channel/impl/email.clj b/src/metabase/channel/impl/email.clj
index 98b2c94184a..7eb30355ff2 100644
--- a/src/metabase/channel/impl/email.clj
+++ b/src/metabase/channel/impl/email.clj
@@ -276,10 +276,12 @@
;; ------------------------------------------------------------------------------------------------;;
(defn- dashboard-pdf-attachment
- "Render the whole dashboard to a PDF email attachment, or `nil` if rendering fails."
- [dashboard-id dashboard-name creator-id parameters]
+ "Render the whole dashboard to a PDF email attachment, or `nil` if rendering fails. `parts` are the dashboard's
+ already-executed parts, reused so the PDF generation doesn't re-run every query."
+ [dashboard-id dashboard-name creator-id parameters parts]
(try
- (let [pdf-bytes (channel.render/render-dashboard-to-pdf dashboard-id creator-id (vec parameters))
+ ;; TODO: (bshepherdson, 2026-07-02) This should not be hard-coding the paper size.
+ (let [pdf-bytes (channel.render/render-dashboard-to-pdf dashboard-id creator-id (vec parameters) :a4 parts)
temp-file (doto (File/createTempFile "metabase_dashboard_" ".pdf")
(.deleteOnExit))]
(with-open [os (io/output-stream temp-file)]
@@ -336,7 +338,7 @@
icon-attachment (make-message-attachment (first (icon-bundle :dashboard)))
card-attachments (map make-message-attachment merged-attachments)
pdf-attachment (when include_pdf
- (dashboard-pdf-attachment (:id dashboard) (:name dashboard) creator_id parameters))
+ (dashboard-pdf-attachment (:id dashboard) (:name dashboard) creator_id parameters dashboard_parts))
attachments (cond-> (into [icon-attachment] result-attachments)
(not attachment_only) (concat card-attachments)
pdf-attachment (concat [pdf-attachment]))
diff --git a/src/metabase/channel/impl/slack.clj b/src/metabase/channel/impl/slack.clj
index 014f6003f95..ba3b21fb1c7 100644
--- a/src/metabase/channel/impl/slack.clj
+++ b/src/metabase/channel/impl/slack.clj
@@ -258,10 +258,11 @@
(parameter-markdown parameter)))))
(defn- dashboard-pdf
- "Render the whole dashboard to a PDF for Slack, returning `{:bytes ... :filename ...}` or `nil` if rendering fails."
- [dashboard creator-id parameters]
+ "Render the whole dashboard to a PDF for Slack, returning `{:bytes ... :filename ...}` or `nil` if rendering fails.
+ `parts` are the dashboard's already-executed parts, reused so the PDF generation doesn't re-run every query."
+ [dashboard creator-id parameters parts]
(try
- (let [pdf-bytes (channel.render/render-dashboard-to-pdf (:id dashboard) creator-id (vec parameters))]
+ (let [pdf-bytes (channel.render/render-dashboard-to-pdf (:id dashboard) creator-id (vec parameters) :a4 parts)]
{:bytes pdf-bytes
:filename (-> dashboard
(some-> :name str/trim)
@@ -277,7 +278,8 @@
(let [all-params (:parameters payload)
top-level-params (impl.util/remove-inline-parameters all-params (:dashboard_parts payload))
dashboard (:dashboard payload)
- pdf (some-> (when include_pdf (dashboard-pdf dashboard creator_id all-params))
+ pdf (some-> (when include_pdf
+ (dashboard-pdf dashboard creator_id all-params (:dashboard_parts payload)))
(assoc :comment (slack-dashboard-caption dashboard (:common_name creator)
all-params top-level-params)))
blocks (if pdf
diff --git a/src/metabase/channel/render/pdf.clj b/src/metabase/channel/render/pdf.clj
index 340d5a73e6a..0b1b8209b9d 100644
--- a/src/metabase/channel/render/pdf.clj
+++ b/src/metabase/channel/render/pdf.clj
@@ -417,18 +417,23 @@
(defn- dashcard->cell
"Build a renderable cell from a dashcard, preserving its grid geometry.
- Returns nil for dashcard kinds we don't render (placeholder/action) or cards that fail/are empty."
- [dashcard parameters]
+ `parts-by-dashcard-id` maps a dashcard id to its already-executed `:card` part -- either the subscription payload's
+ parts (so email/Slack/PDF run each query once) or this render's own single
+ [[metabase.notification.payload.core/execute-dashboard]] call. Queries are never re-run here.
+
+ Returns nil for dashcard kinds we don't render (placeholder/action), or for a card with no part -- a card whose
+ query failed or was hidden while empty (`card.hide_empty`), which `execute-dashboard` omits."
+ [dashcard parameters parts-by-dashcard-id]
(let [inline (resolve-inline-params dashcard parameters)
geom (cond-> (select-keys dashcard [:row :col :size_x :size_y])
(seq inline) (assoc :inline-params inline))]
(cond
(:card_id dashcard)
- ;; Large card results are streamed to disk as a StreamingTempFileStorage (see
- ;; [[metabase.notification.payload.temp-storage]]); realize them here -- exactly as the email/Slack/HTTP
- ;; channels do -- so the renderer sees an ordinary `:rows` collection rather than the disk-backed handle.
- (when-let [part (some-> (notification.payload/execute-dashboard-subscription-card dashcard parameters)
- channel.shared/maybe-realize-data-rows)]
+ ;; Reuse the pre-executed part. Large results stay disk-backed (a StreamingTempFileStorage handle, see
+ ;; [[metabase.notification.payload.temp-storage]]) here and are realized lazily at draw time in
+ ;; [[render-card-cell!]] -- so only the card being drawn is resident in memory, exactly as the email channel
+ ;; walks its parts one at a time to keep its memory watermark low.
+ (when-let [part (get parts-by-dashcard-id (:id dashcard))]
(assoc geom
:kind :card
:part (cond-> part
@@ -456,13 +461,13 @@
(defn- build-cells
"Order a tab's (or the untabbed dashboard's) dashcards and build cells, normalizing row numbers so the first
- row of the section is 0."
- [dashcards parameters]
+ row of the section is 0. `parts-by-dashcard-id` supplies each card's already-executed part (see [[dashcard->cell]])."
+ [dashcards parameters parts-by-dashcard-id]
(let [sorted (sort dashboard-card/dashcard-comparator dashcards)
min-row (if (seq sorted)
(apply min (map :row sorted))
0)]
- (into [] (comp (keep #(dashcard->cell % parameters))
+ (into [] (comp (keep #(dashcard->cell % parameters parts-by-dashcard-id))
(map #(update % :row - min-row)))
sorted)))
@@ -955,12 +960,15 @@
Rectangular (ECharts/visx) charts are then rendered to exactly fill the remaining body area (matching the frontend);
other types render their body title-less and resized (preserving aspect) to fit into the body area."
- [^PDDocument doc ^PDPageContentStream cs timezone
- {:keys [card dashcard inline-params]
- {:keys [data]} :result
- :as part}
- x top-y cell-w cell-h]
- (let [title (card-title card dashcard)
+ [^PDDocument doc ^PDPageContentStream cs timezone part x top-y cell-w cell-h]
+ (let [;; Realize the disk-backed rows for THIS card only, here at draw time (email does the same, one part at a
+ ;; time). A result too large for memory comes back marked `:render/too-large?` instead of loaded. Once this
+ ;; function returns, the realized rows are unreferenced and become garbage before the next card is drawn --
+ ;; so peak memory is one card's rows, not the whole dashboard's.
+ {:keys [card dashcard inline-params]
+ {:keys [data]} :result
+ :as part} (channel.shared/maybe-realize-data-rows part)
+ title (card-title card dashcard)
title-h (typeset/text-block-height (font/face :bold) common/chart-title-pt cell-w (* 0.5 cell-h) title)
;; Inline parameters (filters attached to this card) render between the title and the chart body,
;; like the email subscription does.
@@ -1316,33 +1324,51 @@
"Render the dashboard with `dashboard-id` to PDF bytes, running queries as user `user-id`. `parameters` is a vector
of dashboard parameter overrides; `[]` to use the dashboard's own parameter defaults.
- `paper-key` is `:a4` (default) or `:letter`. Lays cards out following the dashboard's explicit 24-column grid."
+ `paper-key` is `:a4` (default) or `:letter`. Lays cards out following the dashboard's explicit 24-column grid.
+
+ `parts` are the already-executed dashboard parts from a subscription payload (see
+ [[metabase.notification.payload.core/execute-dashboard]]). When supplied they are the source of every card's
+ results, so queries are NOT re-run -- an email or Slack subscription that renders both chart images and a PDF
+ attachment executes each query once. When `parts` is nil (the download API) the queries are executed here via a
+ single `execute-dashboard` call, which shares one spill budget across all cards exactly like an email; the temp
+ files that execution creates are cleaned up before returning."
(^bytes [dashboard-id user-id parameters]
- (render-dashboard-to-pdf dashboard-id user-id parameters :a4))
+ (render-dashboard-to-pdf dashboard-id user-id parameters :a4 nil))
(^bytes [dashboard-id user-id parameters paper-key]
+ (render-dashboard-to-pdf dashboard-id user-id parameters paper-key nil))
+ (^bytes [dashboard-id user-id parameters paper-key parts]
(request/with-current-user user-id
- (let [dims (paper-dims paper-key)
- dash (t2/select-one :model/Dashboard :id dashboard-id)
- tabs (t2/select :model/DashboardTab :dashboard_id dashboard-id {:order-by [[:position :asc]]})
- dcs (t2/hydrate (t2/select :model/DashboardCard :dashboard_id dashboard-id) :card)
+ (let [owned-parts? (nil? parts)
+ ;; Reuse the subscription's parts, or run one shared-budget execution ourselves (matching the email
+ ;; channel). Either way, large results are disk-backed handles at this point -- realized one card at a
+ ;; time at draw time (see [[render-card-cell!]]), never all at once.
+ parts (or parts (notification.payload/execute-dashboard dashboard-id user-id parameters))
+ ;; index the `:card` parts by their dashcard id so build-cells can look each one up by grid position
+ parts-idx (into {} (comp (filter #(= :card (:type %)))
+ (map (juxt #(-> % :dashcard :id) identity)))
+ parts)
+ dims (paper-dims paper-key)
+ dash (t2/select-one :model/Dashboard :id dashboard-id)
+ tabs (t2/select :model/DashboardTab :dashboard_id dashboard-id {:order-by [[:position :asc]]})
+ dcs (t2/hydrate (t2/select :model/DashboardCard :dashboard_id dashboard-id) :card)
;; only treat a dashboard as tabbed when there's more than one tab -- a lone tab isn't shown as a tab in
;; the UI, so the PDF shouldn't draw (or reserve space for) its title either
- tabbed? (> (count tabs) 1)
- by-tab (group-by :dashboard_tab_id dcs)
- resolved (resolve-parameters dash parameters)
- sections (if tabbed?
- (mapv (fn [t]
- {:tab-name (:name t)
- :cells (build-cells (get by-tab (:id t)) resolved)})
- tabs)
- [{:tab-name nil
- :cells (build-cells dcs resolved)}])
- timezone (some (fn [s]
- (some #(when (= :card (:kind %))
- (-> % :part :card render.card/defaulted-timezone))
- (:cells s)))
- sections)
- doc (PDDocument.)]
+ tabbed? (> (count tabs) 1)
+ by-tab (group-by :dashboard_tab_id dcs)
+ resolved (resolve-parameters dash parameters)
+ sections (if tabbed?
+ (mapv (fn [t]
+ {:tab-name (:name t)
+ :cells (build-cells (get by-tab (:id t)) resolved parts-idx)})
+ tabs)
+ [{:tab-name nil
+ :cells (build-cells dcs resolved parts-idx)}])
+ timezone (some (fn [s]
+ (some #(when (= :card (:kind %))
+ (-> % :part :card render.card/defaulted-timezone))
+ (:cells s)))
+ sections)
+ doc (PDDocument.)]
(try
(binding [font/*fonts* (font/load-fonts! doc)
font/*em-width* (memo/memo font/raw-em-width-inner)]
@@ -1359,7 +1385,15 @@
(.save doc os)
(.toByteArray os)))
(finally
- (.close doc)))))))
+ (.close doc)
+ ;; Clean up ONLY the temp files we created. When `parts` were passed in, the notification pipeline
+ ;; ([[metabase.notification.send/do-after-notification-sent]]) owns their cleanup, once every channel has
+ ;; rendered -- deleting them here would pull the rug out from the email/Slack render sharing the same parts.
+ (when owned-parts?
+ (try
+ (run! #(some-> % :result :data :rows notification.payload/cleanup!) parts)
+ (catch Throwable e
+ (log/warn e "Error cleaning up temp files for dashboard PDF"))))))))))
(defn render-dashboard-to-pdf-file
"Convenience wrapper for the REPL: render the dashboard to PDF and write it to `path`."
diff --git a/test/metabase/channel/render/pdf_test.clj b/test/metabase/channel/render/pdf_test.clj
index 8f81e29d0b2..cae102232ce 100644
--- a/test/metabase/channel/render/pdf_test.clj
+++ b/test/metabase/channel/render/pdf_test.clj
@@ -16,7 +16,6 @@
[metabase.channel.render.pdf.font :as font]
[metabase.channel.render.pdf.typeset :as typeset]
[metabase.channel.shared :as channel.shared]
- [metabase.notification.payload.core :as notification.payload]
[metabase.notification.payload.temp-storage :as temp-storage]
[metabase.notification.settings :as notification.settings]
[metabase.test :as mt]
@@ -263,7 +262,7 @@
:size_y 1
:visualization_settings {:virtual_card {:display "link"}
:link {:url "https://example.com"}}}
- []))))))
+ [] nil))))))
(def ^:private iframe-html
"<iframe width=\"560\" src=\"https://www.youtube.com/embed/x\" allowfullscreen></iframe>")
@@ -299,7 +298,7 @@
:size_y 3
:visualization_settings {:virtual_card {:display "iframe"}
:iframe "https://youtu.be/abc"}}
- [])))))
+ [] nil)))))
;; --------------------------------------------------------------------------------------------
;; Parameter name-column sizing (min-column-width)
@@ -757,21 +756,40 @@
(is (thrown? IllegalArgumentException
(render.card/detect-pulse-chart-type {:display "table" :name "t"} nil
{:cols cols :rows storage}))))
- (testing "dashcard->cell realizes the rows into an ordinary vector"
+ (testing "dashcard->cell reuses the pre-executed part from the index, deferring realization (rows stay on disk)"
(let [part {:card {:display "table"
:name "t"}
- :dashcard nil
+ :dashcard {:id 99}
+ :type :card
:result {:data {:cols cols
:rows storage}}}
- dashcard {:card_id 1
+ dashcard {:id 99
+ :card_id 1
:row 0
:col 0
:size_x 6
- :size_y 4}]
- (with-redefs [notification.payload/execute-dashboard-subscription-card (constantly part)]
- (let [realized (get-in (#'pdf/dashcard->cell dashcard []) [:part :result :data :rows])]
- (is (= rows realized))
- (is (not (temp-storage/streaming-temp-file? realized))))))))))
+ :size_y 4}
+ cell (#'pdf/dashcard->cell dashcard [] {99 part})]
+ (is (= :card (:kind cell)))
+ (is (temp-storage/streaming-temp-file? (get-in cell [:part :result :data :rows]))
+ "the disk-backed handle is NOT realized in dashcard->cell -- realization is deferred to draw time")
+ (is (nil? (#'pdf/dashcard->cell dashcard [] {}))
+ "a dashcard with no part in the index (hidden-empty / failed card) is omitted")))
+ (testing "render-card-cell! realizes the disk-backed rows at draw time and draws without throwing"
+ (let [part {:card {:display "table"
+ :name "t"}
+ :dashcard nil
+ :result {:data {:cols cols
+ :rows storage}}}]
+ (with-open [doc (PDDocument.)]
+ (binding [font/*fonts* (#'font/load-fonts! doc)]
+ (let [page (PDPage. PDRectangle/A4)]
+ (.addPage doc page)
+ (with-open [cs (PDPageContentStream. doc page)]
+ (#'pdf/render-card-cell! doc cs nil part 36.0 760.0 480.0 360.0))
+ (let [baos (ByteArrayOutputStream.)]
+ (.save doc baos)
+ (is (pos? (count (.toByteArray baos)))))))))))))
(deftest ^:parallel too-large-result-marked-test
(testing "maybe-realize-data-rows marks an oversized disk-spilled result :render/too-large? instead of loading it"
diff --git a/test/metabase/pulse/dashboard_subscription_test.clj b/test/metabase/pulse/dashboard_subscription_test.clj
index 564aa5ef2b5..000114cf334 100644
--- a/test/metabase/pulse/dashboard_subscription_test.clj
+++ b/test/metabase/pulse/dashboard_subscription_test.clj
@@ -1395,10 +1395,11 @@
(let [render-args (atom nil)]
;; Stub the renderer: avoid producing a real PDF, and capture the args it's called with.
(with-redefs [channel.render/render-dashboard-to-pdf
- (fn [dashboard-id user-id parameters]
+ (fn [dashboard-id user-id parameters & [_paper-key parts]]
(reset! render-args {:dashboard-id dashboard-id
:user-id user-id
- :parameters parameters})
+ :parameters parameters
+ :parts parts})
(.getBytes "%PDF-1.4 stub" "UTF-8"))]
(mt/with-temp [:model/Card {card-id :id} {:name pulse.test-util/card-name
:dataset_query (mt/mbql-query orders {:limit 1})}
@@ -1420,6 +1421,9 @@
(testing "renderer is called with the subscription's dashboard id and resolved parameters"
(is (= dashboard-id (:dashboard-id @render-args)))
(is (= [] (:parameters @render-args))))
+ (testing "the already-executed dashboard parts are handed to the renderer so queries aren't re-run"
+ (is (some #(= :card (:type %))
+ (:parts @render-args))))
(let [message (:message (first (:channel/email pulse-results)))
pdf-part (some #(when (= "application/pdf" (:content-type %)) %) message)]
(testing "the PDF attachment is named after the dashboard"
@@ -1473,8 +1477,9 @@
:details {:channel "#general" :include_pdf true}}]
(let [render-args (atom nil)]
(with-redefs [channel.render/render-dashboard-to-pdf
- (fn [dashboard-id user-id parameters]
- (reset! render-args {:dashboard-id dashboard-id :user-id user-id :parameters parameters})
+ (fn [dashboard-id user-id parameters & [_paper-key parts]]
+ (reset! render-args {:dashboard-id dashboard-id :user-id user-id
+ :parameters parameters :parts parts})
(.getBytes "%PDF-1.4 stub" "UTF-8"))]
(pulse.test-util/slack-test-setup!
(let [results (pulse.test-util/with-captured-channel-send-messages!