The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[PDFs] Reuse query results and PNG rendering memory pools for PDFs (#76918)
Braden Shepherdsonb8d634be905d
diff --git a/src/metabase/channel/impl/email.clj b/src/metabase/channel/impl/email.cljindex 98b2c94184a..7eb30355ff2 100644--- a/src/metabase/channel/impl/email.clj+++ b/src/metabase/channel/impl/email.clj@@ -276,10 +276,12 @@;; ------------------------------------------------------------------------------------------------;;(defn- dashboard-pdf-attachment- "Render the whole dashboard to a PDF email attachment, or `nil` if rendering fails."- [dashboard-id dashboard-name creator-id parameters]+ "Render the whole dashboard to a PDF email attachment, or `nil` if rendering fails. `parts` are the dashboard's+ already-executed parts, reused so the PDF generation doesn't re-run every query."+ [dashboard-id dashboard-name creator-id parameters parts](try- (let [pdf-bytes (channel.render/render-dashboard-to-pdf dashboard-id creator-id (vec parameters))+ ;; TODO: (bshepherdson, 2026-07-02) This should not be hard-coding the paper size.+ (let [pdf-bytes (channel.render/render-dashboard-to-pdf dashboard-id creator-id (vec parameters) :a4 parts)temp-file (doto (File/createTempFile "metabase_dashboard_" ".pdf")(.deleteOnExit))](with-open [os (io/output-stream temp-file)]@@ -336,7 +338,7 @@icon-attachment (make-message-attachment (first (icon-bundle :dashboard)))card-attachments (map make-message-attachment merged-attachments)pdf-attachment (when include_pdf- (dashboard-pdf-attachment (:id dashboard) (:name dashboard) creator_id parameters))+ (dashboard-pdf-attachment (:id dashboard) (:name dashboard) creator_id parameters dashboard_parts))attachments (cond-> (into [icon-attachment] result-attachments)(not attachment_only) (concat card-attachments)pdf-attachment (concat [pdf-attachment]))diff --git a/src/metabase/channel/impl/slack.clj b/src/metabase/channel/impl/slack.cljindex 014f6003f95..ba3b21fb1c7 100644--- a/src/metabase/channel/impl/slack.clj+++ b/src/metabase/channel/impl/slack.clj@@ -258,10 +258,11 @@(parameter-markdown parameter)))))(defn- dashboard-pdf- "Render the whole dashboard to a PDF for Slack, returning `{:bytes ... :filename ...}` or `nil` if rendering fails."- [dashboard creator-id parameters]+ "Render the whole dashboard to a PDF for Slack, returning `{:bytes ... :filename ...}` or `nil` if rendering fails.+ `parts` are the dashboard's already-executed parts, reused so the PDF generation doesn't re-run every query."+ [dashboard creator-id parameters parts](try- (let [pdf-bytes (channel.render/render-dashboard-to-pdf (:id dashboard) creator-id (vec parameters))]+ (let [pdf-bytes (channel.render/render-dashboard-to-pdf (:id dashboard) creator-id (vec parameters) :a4 parts)]{:bytes pdf-bytes:filename (-> dashboard(some-> :name str/trim)@@ -277,7 +278,8 @@(let [all-params (:parameters payload)top-level-params (impl.util/remove-inline-parameters all-params (:dashboard_parts payload))dashboard (:dashboard payload)- pdf (some-> (when include_pdf (dashboard-pdf dashboard creator_id all-params))+ pdf (some-> (when include_pdf+ (dashboard-pdf dashboard creator_id all-params (:dashboard_parts payload)))(assoc :comment (slack-dashboard-caption dashboard (:common_name creator)all-params top-level-params)))blocks (if pdfdiff --git a/src/metabase/channel/render/pdf.clj b/src/metabase/channel/render/pdf.cljindex 340d5a73e6a..0b1b8209b9d 100644--- a/src/metabase/channel/render/pdf.clj+++ b/src/metabase/channel/render/pdf.clj@@ -417,18 +417,23 @@(defn- dashcard->cell"Build a renderable cell from a dashcard, preserving its grid geometry.- Returns nil for dashcard kinds we don't render (placeholder/action) or cards that fail/are empty."- [dashcard parameters]+ `parts-by-dashcard-id` maps a dashcard id to its already-executed `:card` part -- either the subscription payload's+ parts (so email/Slack/PDF run each query once) or this render's own single+ [[metabase.notification.payload.core/execute-dashboard]] call. Queries are never re-run here.++ Returns nil for dashcard kinds we don't render (placeholder/action), or for a card with no part -- a card whose+ query failed or was hidden while empty (`card.hide_empty`), which `execute-dashboard` omits."+ [dashcard parameters parts-by-dashcard-id](let [inline (resolve-inline-params dashcard parameters)geom (cond-> (select-keys dashcard [:row :col :size_x :size_y])(seq inline) (assoc :inline-params inline))](cond(:card_id dashcard)- ;; Large card results are streamed to disk as a StreamingTempFileStorage (see- ;; [[metabase.notification.payload.temp-storage]]); realize them here -- exactly as the email/Slack/HTTP- ;; channels do -- so the renderer sees an ordinary `:rows` collection rather than the disk-backed handle.- (when-let [part (some-> (notification.payload/execute-dashboard-subscription-card dashcard parameters)- channel.shared/maybe-realize-data-rows)]+ ;; Reuse the pre-executed part. Large results stay disk-backed (a StreamingTempFileStorage handle, see+ ;; [[metabase.notification.payload.temp-storage]]) here and are realized lazily at draw time in+ ;; [[render-card-cell!]] -- so only the card being drawn is resident in memory, exactly as the email channel+ ;; walks its parts one at a time to keep its memory watermark low.+ (when-let [part (get parts-by-dashcard-id (:id dashcard))](assoc geom:kind :card:part (cond-> part@@ -456,13 +461,13 @@(defn- build-cells"Order a tab's (or the untabbed dashboard's) dashcards and build cells, normalizing row numbers so the first- row of the section is 0."- [dashcards parameters]+ row of the section is 0. `parts-by-dashcard-id` supplies each card's already-executed part (see [[dashcard->cell]])."+ [dashcards parameters parts-by-dashcard-id](let [sorted (sort dashboard-card/dashcard-comparator dashcards)min-row (if (seq sorted)(apply min (map :row sorted))0)]- (into [] (comp (keep #(dashcard->cell % parameters))+ (into [] (comp (keep #(dashcard->cell % parameters parts-by-dashcard-id))(map #(update % :row - min-row)))sorted)))@@ -955,12 +960,15 @@Rectangular (ECharts/visx) charts are then rendered to exactly fill the remaining body area (matching the frontend);other types render their body title-less and resized (preserving aspect) to fit into the body area."- [^PDDocument doc ^PDPageContentStream cs timezone- {:keys [card dashcard inline-params]- {:keys [data]} :result- :as part}- x top-y cell-w cell-h]- (let [title (card-title card dashcard)+ [^PDDocument doc ^PDPageContentStream cs timezone part x top-y cell-w cell-h]+ (let [;; Realize the disk-backed rows for THIS card only, here at draw time (email does the same, one part at a+ ;; time). A result too large for memory comes back marked `:render/too-large?` instead of loaded. Once this+ ;; function returns, the realized rows are unreferenced and become garbage before the next card is drawn --+ ;; so peak memory is one card's rows, not the whole dashboard's.+ {:keys [card dashcard inline-params]+ {:keys [data]} :result+ :as part} (channel.shared/maybe-realize-data-rows part)+ title (card-title card dashcard)title-h (typeset/text-block-height (font/face :bold) common/chart-title-pt cell-w (* 0.5 cell-h) title);; Inline parameters (filters attached to this card) render between the title and the chart body,;; like the email subscription does.@@ -1316,33 +1324,51 @@"Render the dashboard with `dashboard-id` to PDF bytes, running queries as user `user-id`. `parameters` is a vectorof dashboard parameter overrides; `[]` to use the dashboard's own parameter defaults.- `paper-key` is `:a4` (default) or `:letter`. Lays cards out following the dashboard's explicit 24-column grid."+ `paper-key` is `:a4` (default) or `:letter`. Lays cards out following the dashboard's explicit 24-column grid.++ `parts` are the already-executed dashboard parts from a subscription payload (see+ [[metabase.notification.payload.core/execute-dashboard]]). When supplied they are the source of every card's+ results, so queries are NOT re-run -- an email or Slack subscription that renders both chart images and a PDF+ attachment executes each query once. When `parts` is nil (the download API) the queries are executed here via a+ single `execute-dashboard` call, which shares one spill budget across all cards exactly like an email; the temp+ files that execution creates are cleaned up before returning."(^bytes [dashboard-id user-id parameters]- (render-dashboard-to-pdf dashboard-id user-id parameters :a4))+ (render-dashboard-to-pdf dashboard-id user-id parameters :a4 nil))(^bytes [dashboard-id user-id parameters paper-key]+ (render-dashboard-to-pdf dashboard-id user-id parameters paper-key nil))+ (^bytes [dashboard-id user-id parameters paper-key parts](request/with-current-user user-id- (let [dims (paper-dims paper-key)- dash (t2/select-one :model/Dashboard :id dashboard-id)- tabs (t2/select :model/DashboardTab :dashboard_id dashboard-id {:order-by [[:position :asc]]})- dcs (t2/hydrate (t2/select :model/DashboardCard :dashboard_id dashboard-id) :card)+ (let [owned-parts? (nil? parts)+ ;; Reuse the subscription's parts, or run one shared-budget execution ourselves (matching the email+ ;; channel). Either way, large results are disk-backed handles at this point -- realized one card at a+ ;; time at draw time (see [[render-card-cell!]]), never all at once.+ parts (or parts (notification.payload/execute-dashboard dashboard-id user-id parameters))+ ;; index the `:card` parts by their dashcard id so build-cells can look each one up by grid position+ parts-idx (into {} (comp (filter #(= :card (:type %)))+ (map (juxt #(-> % :dashcard :id) identity)))+ parts)+ dims (paper-dims paper-key)+ dash (t2/select-one :model/Dashboard :id dashboard-id)+ tabs (t2/select :model/DashboardTab :dashboard_id dashboard-id {:order-by [[:position :asc]]})+ dcs (t2/hydrate (t2/select :model/DashboardCard :dashboard_id dashboard-id) :card);; only treat a dashboard as tabbed when there's more than one tab -- a lone tab isn't shown as a tab in;; the UI, so the PDF shouldn't draw (or reserve space for) its title either- tabbed? (> (count tabs) 1)- by-tab (group-by :dashboard_tab_id dcs)- resolved (resolve-parameters dash parameters)- sections (if tabbed?- (mapv (fn [t]- {:tab-name (:name t)- :cells (build-cells (get by-tab (:id t)) resolved)})- tabs)- [{:tab-name nil- :cells (build-cells dcs resolved)}])- timezone (some (fn [s]- (some #(when (= :card (:kind %))- (-> % :part :card render.card/defaulted-timezone))- (:cells s)))- sections)- doc (PDDocument.)]+ tabbed? (> (count tabs) 1)+ by-tab (group-by :dashboard_tab_id dcs)+ resolved (resolve-parameters dash parameters)+ sections (if tabbed?+ (mapv (fn [t]+ {:tab-name (:name t)+ :cells (build-cells (get by-tab (:id t)) resolved parts-idx)})+ tabs)+ [{:tab-name nil+ :cells (build-cells dcs resolved parts-idx)}])+ timezone (some (fn [s]+ (some #(when (= :card (:kind %))+ (-> % :part :card render.card/defaulted-timezone))+ (:cells s)))+ sections)+ doc (PDDocument.)](try(binding [font/*fonts* (font/load-fonts! doc)font/*em-width* (memo/memo font/raw-em-width-inner)]@@ -1359,7 +1385,15 @@(.save doc os)(.toByteArray os)))(finally- (.close doc)))))))+ (.close doc)+ ;; Clean up ONLY the temp files we created. When `parts` were passed in, the notification pipeline+ ;; ([[metabase.notification.send/do-after-notification-sent]]) owns their cleanup, once every channel has+ ;; rendered -- deleting them here would pull the rug out from the email/Slack render sharing the same parts.+ (when owned-parts?+ (try+ (run! #(some-> % :result :data :rows notification.payload/cleanup!) parts)+ (catch Throwable e+ (log/warn e "Error cleaning up temp files for dashboard PDF"))))))))))(defn render-dashboard-to-pdf-file"Convenience wrapper for the REPL: render the dashboard to PDF and write it to `path`."diff --git a/test/metabase/channel/render/pdf_test.clj b/test/metabase/channel/render/pdf_test.cljindex 8f81e29d0b2..cae102232ce 100644--- a/test/metabase/channel/render/pdf_test.clj+++ b/test/metabase/channel/render/pdf_test.clj@@ -16,7 +16,6 @@[metabase.channel.render.pdf.font :as font][metabase.channel.render.pdf.typeset :as typeset][metabase.channel.shared :as channel.shared]- [metabase.notification.payload.core :as notification.payload][metabase.notification.payload.temp-storage :as temp-storage][metabase.notification.settings :as notification.settings][metabase.test :as mt]@@ -263,7 +262,7 @@:size_y 1:visualization_settings {:virtual_card {:display "link"}:link {:url "https://example.com"}}}- []))))))+ [] nil))))))(def ^:private iframe-html"<iframe width=\"560\" src=\"https://www.youtube.com/embed/x\" allowfullscreen></iframe>")@@ -299,7 +298,7 @@:size_y 3:visualization_settings {:virtual_card {:display "iframe"}:iframe "https://youtu.be/abc"}}- [])))))+ [] nil)))));; --------------------------------------------------------------------------------------------;; Parameter name-column sizing (min-column-width)@@ -757,21 +756,40 @@(is (thrown? IllegalArgumentException(render.card/detect-pulse-chart-type {:display "table" :name "t"} nil{:cols cols :rows storage}))))- (testing "dashcard->cell realizes the rows into an ordinary vector"+ (testing "dashcard->cell reuses the pre-executed part from the index, deferring realization (rows stay on disk)"(let [part {:card {:display "table":name "t"}- :dashcard nil+ :dashcard {:id 99}+ :type :card:result {:data {:cols cols:rows storage}}}- dashcard {:card_id 1+ dashcard {:id 99+ :card_id 1:row 0:col 0:size_x 6- :size_y 4}]- (with-redefs [notification.payload/execute-dashboard-subscription-card (constantly part)]- (let [realized (get-in (#'pdf/dashcard->cell dashcard []) [:part :result :data :rows])]- (is (= rows realized))- (is (not (temp-storage/streaming-temp-file? realized))))))))))+ :size_y 4}+ cell (#'pdf/dashcard->cell dashcard [] {99 part})]+ (is (= :card (:kind cell)))+ (is (temp-storage/streaming-temp-file? (get-in cell [:part :result :data :rows]))+ "the disk-backed handle is NOT realized in dashcard->cell -- realization is deferred to draw time")+ (is (nil? (#'pdf/dashcard->cell dashcard [] {}))+ "a dashcard with no part in the index (hidden-empty / failed card) is omitted")))+ (testing "render-card-cell! realizes the disk-backed rows at draw time and draws without throwing"+ (let [part {:card {:display "table"+ :name "t"}+ :dashcard nil+ :result {:data {:cols cols+ :rows storage}}}]+ (with-open [doc (PDDocument.)]+ (binding [font/*fonts* (#'font/load-fonts! doc)]+ (let [page (PDPage. PDRectangle/A4)]+ (.addPage doc page)+ (with-open [cs (PDPageContentStream. doc page)]+ (#'pdf/render-card-cell! doc cs nil part 36.0 760.0 480.0 360.0))+ (let [baos (ByteArrayOutputStream.)]+ (.save doc baos)+ (is (pos? (count (.toByteArray baos)))))))))))))(deftest ^:parallel too-large-result-marked-test(testing "maybe-realize-data-rows marks an oversized disk-spilled result :render/too-large? instead of loading it"diff --git a/test/metabase/pulse/dashboard_subscription_test.clj b/test/metabase/pulse/dashboard_subscription_test.cljindex 564aa5ef2b5..000114cf334 100644--- a/test/metabase/pulse/dashboard_subscription_test.clj+++ b/test/metabase/pulse/dashboard_subscription_test.clj@@ -1395,10 +1395,11 @@(let [render-args (atom nil)];; Stub the renderer: avoid producing a real PDF, and capture the args it's called with.(with-redefs [channel.render/render-dashboard-to-pdf- (fn [dashboard-id user-id parameters]+ (fn [dashboard-id user-id parameters & [_paper-key parts]](reset! render-args {:dashboard-id dashboard-id:user-id user-id- :parameters parameters})+ :parameters parameters+ :parts parts})(.getBytes "%PDF-1.4 stub" "UTF-8"))](mt/with-temp [:model/Card {card-id :id} {:name pulse.test-util/card-name:dataset_query (mt/mbql-query orders {:limit 1})}@@ -1420,6 +1421,9 @@(testing "renderer is called with the subscription's dashboard id and resolved parameters"(is (= dashboard-id (:dashboard-id @render-args)))(is (= [] (:parameters @render-args))))+ (testing "the already-executed dashboard parts are handed to the renderer so queries aren't re-run"+ (is (some #(= :card (:type %))+ (:parts @render-args))))(let [message (:message (first (:channel/email pulse-results)))pdf-part (some #(when (= "application/pdf" (:content-type %)) %) message)](testing "the PDF attachment is named after the dashboard"@@ -1473,8 +1477,9 @@:details {:channel "#general" :include_pdf true}}](let [render-args (atom nil)](with-redefs [channel.render/render-dashboard-to-pdf- (fn [dashboard-id user-id parameters]- (reset! render-args {:dashboard-id dashboard-id :user-id user-id :parameters parameters})+ (fn [dashboard-id user-id parameters & [_paper-key parts]]+ (reset! render-args {:dashboard-id dashboard-id :user-id user-id+ :parameters parameters :parts parts})(.getBytes "%PDF-1.4 stub" "UTF-8"))](pulse.test-util/slack-test-setup!(let [results (pulse.test-util/with-captured-channel-send-messages!