Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[PDFs] Resolve large results in temp files; surface "too large" errors (#76605)

Braden Shepherdson0161c22652dc
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 58d115d4441..11e80e5c4c3 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -181,6 +181,10 @@
clojure.core.async/to-chan!!
metabase.api.macros.defendpoint.tools-manifest/assert-optional-fields-nullable!
metabase.channel.core/send!
+ metabase.channel.render.pdf/draw-too-large!
+ metabase.channel.render.pdf/draw-no-results!
+ metabase.channel.render.pdf/render-card-cell!
+ metabase.channel.render.pdf.font/load-fonts!
metabase.warehouses.models.database/assert-not-h2!
metabase.driver.sql-jdbc.execute/execute-prepared-statement!
metabase.notification.models/create-notification!
@@ -803,7 +807,6 @@
;;
{:pattern "^metabase(?:((?:-enterprise\\.sandbox)?\\.query-processor)|(?:\\.driver\\.sql(?!server)(?!ite))).*-test$"
:name general-qp-and-driver-test-namespaces}
-
{:pattern "(?:^mage.*)"
:name mage-namespaces}]
diff --git a/src/metabase/channel/render/pdf.clj b/src/metabase/channel/render/pdf.clj
index 6b4458b0ca3..340d5a73e6a 100644
--- a/src/metabase/channel/render/pdf.clj
+++ b/src/metabase/channel/render/pdf.clj
@@ -48,6 +48,7 @@
[metabase.channel.render.pdf.typeset :as typeset]
[metabase.channel.render.png :as png]
[metabase.channel.render.util :as render.util]
+ [metabase.channel.shared :as channel.shared]
[metabase.dashboards.models.dashboard-card :as dashboard-card]
[metabase.notification.payload.core :as notification.payload]
[metabase.parameters.shared :as shared.params]
@@ -423,7 +424,11 @@
(seq inline) (assoc :inline-params inline))]
(cond
(:card_id dashcard)
- (when-let [part (notification.payload/execute-dashboard-subscription-card dashcard parameters)]
+ ;; Large card results are streamed to disk as a StreamingTempFileStorage (see
+ ;; [[metabase.notification.payload.temp-storage]]); realize them here -- exactly as the email/Slack/HTTP
+ ;; channels do -- so the renderer sees an ordinary `:rows` collection rather than the disk-backed handle.
+ (when-let [part (some-> (notification.payload/execute-dashboard-subscription-card dashcard parameters)
+ channel.shared/maybe-realize-data-rows)]
(assoc geom
:kind :card
:part (cond-> part
@@ -929,6 +934,20 @@
(draw-line! cs (font/face :regular) label-pt label-x label-baseline label)
(.setNonStrokingColor cs Color/BLACK)))
+(defn- draw-too-large!
+ "Draw a centered message when a card's result was too large to load into memory: the disk-spilled result exceeded
+ [[metabase.notification.settings/notification-temp-file-size-max-bytes]], so `maybe-realize-data-rows` skipped the
+ load and marked the part `:render/too-large?` (carrying a localized `:error` string) instead of handing back rows.
+ Drawn as centered native text -- mirroring [[draw-no-results!]] -- so the card reads as a clear explanation rather
+ than the misleading empty state it would otherwise fall through to."
+ [^PDPageContentStream cs x body-top cell-w body-h message]
+ (let [pt common/text-card-pt
+ text (str message)
+ ;; wrap to the body width, then vertically center the resulting block in the body band
+ block-h (typeset/text-block-height (font/face :regular) pt cell-w body-h text)
+ top-y (common/v-center body-top body-h block-h)]
+ (draw-text-block! cs (font/face :regular) pt common/body-color x top-y cell-w body-h text)))
+
(defn- render-card-cell!
"Render a chart/query card into its cell rectangle. The title is drawn natively at the PDF level (crisp text) and
the space it consumes is reserved before the body is rendered. Card descriptions are intentionally omitted -- they
@@ -964,7 +983,10 @@
;; rows (e.g. filtered out) detects as :empty regardless of display, and gets the no-results placeholder.
chart-type (render.card/detect-pulse-chart-type card dashcard data)
table? (= :table chart-type)
- empty? (= :empty chart-type)]
+ empty? (= :empty chart-type)
+ ;; the result was too big to load into memory (see [[draw-too-large!]]); show its message instead of a chart.
+ ;; carries no :data, so it would otherwise detect as :empty and show a misleading no-results placeholder.
+ too-large? (get-in part [:result :render/too-large?])]
;; Debug overlays (drawn first, so content sits on top): the full title box (blue) and the full
;; chart-body box (red) -- the *allocated* space, regardless of how much the content fills.
(when *debug-boxes*
@@ -981,6 +1003,12 @@
(binding [js.svg/*svg-background-color* (when-not *debug-boxes*
js.svg/*svg-background-color*)]
(b/cond
+ ;; result too large to load: show its explanatory message rather than attempting (and failing) to draw a chart
+ too-large?
+ (draw-too-large! cs x body-top cell-w body-h
+ (or (get-in part [:result :error])
+ (tru "Results too large to display.")))
+
;; no rows: show the centered no-results placeholder instead of attempting (and failing) to draw a chart
empty?
(draw-no-results! doc cs x body-top cell-w body-h)
diff --git a/test/metabase/channel/render/pdf_test.clj b/test/metabase/channel/render/pdf_test.clj
index ad5543866f5..8f81e29d0b2 100644
--- a/test/metabase/channel/render/pdf_test.clj
+++ b/test/metabase/channel/render/pdf_test.clj
@@ -15,7 +15,11 @@
[metabase.channel.render.pdf :as pdf]
[metabase.channel.render.pdf.font :as font]
[metabase.channel.render.pdf.typeset :as typeset]
- [metabase.test.util.dynamic-redefs :as dynamic-redefs]
+ [metabase.channel.shared :as channel.shared]
+ [metabase.notification.payload.core :as notification.payload]
+ [metabase.notification.payload.temp-storage :as temp-storage]
+ [metabase.notification.settings :as notification.settings]
+ [metabase.test :as mt]
[metabase.util.memoize :as memo])
(:import
(java.awt Color)
@@ -498,12 +502,12 @@
{:font <face-id> :pt :x :y :text} with coordinates rounded to 0.1pt."
[render!]
(let [calls (atom [])]
- (dynamic-redefs/with-dynamic-fn-redefs [pdf/draw-line! (fn [_cs face font-pt x y text]
- (swap! calls conj {:font (:id face)
- :pt (round1 font-pt)
- :x (round1 x)
- :y (round1 y)
- :text text}))]
+ (mt/with-dynamic-fn-redefs [pdf/draw-line! (fn [_cs face font-pt x y text]
+ (swap! calls conj {:font (:id face)
+ :pt (round1 font-pt)
+ :x (round1 x)
+ :y (round1 y)
+ :text text}))]
(render!))
@calls))
@@ -728,3 +732,82 @@
(let [baos (ByteArrayOutputStream.)]
(.save doc baos)
(is (pos? (count (.toByteArray baos))))))))))))
+
+(defn- ->disk-spilled-rows
+ "Reduce `rows` through the notification rff with a 1-cell budget so they immediately spill to disk, returning the
+ `StreamingTempFileStorage` handle a large subscription query produces (see
+ [[metabase.notification.payload.temp-storage]])."
+ [cols rows]
+ (let [budget (temp-storage/make-resident-budget {:per-card 1 :resident-cap 1 :floor 1})
+ rf ((temp-storage/notification-rff {:budget budget}) {:cols cols})]
+ (get-in (rf (reduce rf (rf) rows)) [:data :rows])))
+
+(deftest ^:synchronized disk-spilled-rows-realized-test
+ (testing "a card whose large result spilled to disk is realized before rendering, instead of crashing on the handle"
+ ;; UXW-4614 regression: large subscription queries stream rows to a StreamingTempFileStorage; the PDF path must
+ ;; realize them (like the email/Slack/HTTP channels) so the renderer sees an ordinary `:rows` collection.
+ (let [cols [{:name "x"
+ :display_name "X"
+ :base_type :type/Integer}]
+ rows [[1] [2] [3]]
+ storage (->disk-spilled-rows cols rows)]
+ (is (temp-storage/streaming-temp-file? storage)
+ "sanity check: the helper produced a disk-backed handle")
+ (testing "the un-realized handle is exactly what used to blow up detect-pulse-chart-type"
+ (is (thrown? IllegalArgumentException
+ (render.card/detect-pulse-chart-type {:display "table" :name "t"} nil
+ {:cols cols :rows storage}))))
+ (testing "dashcard->cell realizes the rows into an ordinary vector"
+ (let [part {:card {:display "table"
+ :name "t"}
+ :dashcard nil
+ :result {:data {:cols cols
+ :rows storage}}}
+ dashcard {:card_id 1
+ :row 0
+ :col 0
+ :size_x 6
+ :size_y 4}]
+ (with-redefs [notification.payload/execute-dashboard-subscription-card (constantly part)]
+ (let [realized (get-in (#'pdf/dashcard->cell dashcard []) [:part :result :data :rows])]
+ (is (= rows realized))
+ (is (not (temp-storage/streaming-temp-file? realized))))))))))
+
+(deftest ^:parallel too-large-result-marked-test
+ (testing "maybe-realize-data-rows marks an oversized disk-spilled result :render/too-large? instead of loading it"
+ ;; UXW-4614: when the spill file exceeds the size cap, the rows must NOT be read into memory; the part is tagged
+ ;; with a localized :error and :render/too-large? so the renderer can show a message rather than crash or OOM.
+ (let [storage (->disk-spilled-rows [{:name "x"}] [[1] [2] [3]])
+ part {:card {:display "table"}
+ :result {:data {:cols [{:name "x"}]
+ :rows storage}}}]
+ ;; force "too large": cap the loadable size at 1 byte so even this tiny spill is refused
+ (mt/with-dynamic-fn-redefs [notification.settings/notification-temp-file-size-max-bytes (constantly 1)]
+ (let [realized (channel.shared/maybe-realize-data-rows part)]
+ (is (true? (get-in realized [:result :render/too-large?])))
+ (is (string? (get-in realized [:result :error]))))))))
+
+(deftest ^:parallel too-large-card-render-test
+ (testing "a card whose result was too large to load shows its message, not the misleading no-results placeholder"
+ ;; UXW-4614: a :render/too-large? part carries no :data, so it would otherwise detect as :empty; render-card-cell!
+ ;; must take the too-large branch and surface the localized :error instead.
+ (let [part {:card {:display "table" :name "Big table"} :dashcard nil
+ :result {:error "Results too large to display. The query returned too much data to show in this notification."
+ :render/too-large? true
+ :max-size-human-readable "10.0 mb"}}
+ too-large-msg (atom nil)
+ no-results-called (atom false)]
+ (mt/with-dynamic-fn-redefs [pdf/draw-too-large! (fn [_cs _x _bt _cw _bh message]
+ (reset! too-large-msg message))
+ pdf/draw-no-results! (fn [& _]
+ (reset! no-results-called true))]
+ (with-open [doc (PDDocument.)]
+ (binding [font/*fonts* (#'font/load-fonts! doc)]
+ (let [page (PDPage. PDRectangle/A4)]
+ (.addPage doc page)
+ (with-open [cs (PDPageContentStream. doc page)]
+ (#'pdf/render-card-cell! doc cs nil part 36.0 760.0 480.0 360.0))))))
+ (testing "the too-large branch fired with the part's error message"
+ (is (= (get-in part [:result :error]) @too-large-msg)))
+ (testing "and the no-results placeholder was not used"
+ (is (false? @no-results-called))))))