Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

use disk at query time of notification payload creation (#64390)

dpsuttondb11feef9b73
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index ec5bdc350da..5fcbc248c8c 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -364,6 +364,10 @@
(prometheus/counter :metabase-notification/send-error
{:description "Number of errors when sending notifications."
:labels [:payload-type]})
+ (prometheus/counter :metabase-notification/temp-storage
+ {:description "Number and type of temporary storage uses"
+ ;; memory, disk, above-threshold, truncated, not-limited
+ :labels [:storage]})
(prometheus/histogram :metabase-notification/wait-duration-ms
{:description "Duration in milliseconds that notifications wait in the processing queue before being picked up for delivery."
:labels [:payload-type]
diff --git a/src/metabase/channel/email/result_attachment.clj b/src/metabase/channel/email/result_attachment.clj
index f5343408fe1..58d515be2e6 100644
--- a/src/metabase/channel/email/result_attachment.clj
+++ b/src/metabase/channel/email/result_attachment.clj
@@ -89,21 +89,23 @@
dashcard :dashcard
result :result
:as part}]
- (when (pos-int? (:row_count result))
+ (when (and (or (:include_csv card) (:include_xls card))
+ (pos-int? (:row_count result)))
(let [maybe-realize-data-rows (requiring-resolve 'metabase.channel.shared/maybe-realize-data-rows)
result (:result (maybe-realize-data-rows part))
visualizer-title (when (and dashcard (get-in dashcard [:visualization_settings :visualization]))
(not-empty (get-in dashcard [:visualization_settings :visualization :settings :card.title])))
filename-prefix (or visualizer-title original-card-name)]
- (->>
- [(when-let [temp-file (and (:include_csv card)
- (create-temp-file-or-throw! "csv"))]
- (with-open [os (io/output-stream temp-file)]
- (stream-api-results-to-export-format! os {:export-format :csv :format-rows? format-rows :pivot? pivot-results} result))
- (create-result-attachment-map "csv" filename-prefix temp-file))
- (when-let [temp-file (and (:include_xls card)
- (create-temp-file-or-throw! "xlsx"))]
- (with-open [os (io/output-stream temp-file)]
- (stream-api-results-to-export-format! os {:export-format :xlsx :format-rows? format-rows :pivot? pivot-results} result))
- (create-result-attachment-map "xlsx" filename-prefix temp-file))]
- (filterv some?)))))
+ (when-not (:render/too-large? result)
+ (->>
+ [(when-let [temp-file (and (:include_csv card)
+ (create-temp-file-or-throw! "csv"))]
+ (with-open [os (io/output-stream temp-file)]
+ (stream-api-results-to-export-format! os {:export-format :csv :format-rows? format-rows :pivot? pivot-results} result))
+ (create-result-attachment-map "csv" filename-prefix temp-file))
+ (when-let [temp-file (and (:include_xls card)
+ (create-temp-file-or-throw! "xlsx"))]
+ (with-open [os (io/output-stream temp-file)]
+ (stream-api-results-to-export-format! os {:export-format :xlsx :format-rows? format-rows :pivot? pivot-results} result))
+ (create-result-attachment-map "xlsx" filename-prefix temp-file))]
+ (filterv some?))))))
diff --git a/src/metabase/channel/render/body.clj b/src/metabase/channel/render/body.clj
index 54a20107794..8f66f0ca710 100644
--- a/src/metabase/channel/render/body.clj
+++ b/src/metabase/channel/render/body.clj
@@ -639,6 +639,21 @@
[_chart-type _render-type _timezone-id _card _dashcard _data]
@card-error-rendered-info)
+(mu/defmethod render :card-error/results-too-large :- ::RenderedPartCard
+ [_chart-type _render-type _timezone-id _card _dashcard data]
+ (let [filesize-limit (:max-size-human-readable data "10.0 mb")]
+ {:attachments nil,
+ :content [:div
+ {:style "font-family: Lato, \"Helvetica Neue\", Helvetica, Arial, sans-serif; margin: 0px 0;"}
+ [:div
+ {:style "background-color: #F9FBFC; border: 1px solid #DCE1E4; border-radius: 8px; padding: 16px;"}
+ [:p
+ {:style "margin: 4px; font-size: 14px; font-weight: 700; color: #4C5773;"}
+ (trs "This chart exceeded the {0} size limit." filesize-limit)]
+ [:p
+ {:style "margin: 4px; font-size: 12px; font-weight: 400; color: #696E7B;"}
+ (trs "You can make it smaller by adding filters, or summarizing the data.")]]]}))
+
(mu/defmethod render :render-error :- ::RenderedPartCard
[_chart-type _render-type _timezone-id _card _dashcard _data]
@error-rendered-info)
diff --git a/src/metabase/channel/render/card.clj b/src/metabase/channel/render/card.clj
index 12d6aba8d11..c34bc9ae6a2 100644
--- a/src/metabase/channel/render/card.clj
+++ b/src/metabase/channel/render/card.clj
@@ -153,13 +153,20 @@
(log/debugf "Rendering pulse card with chart-type %s and render-type %s" chart-type render-type)
(body/render chart-type render-type timezone-id card dashcard data))
(catch Throwable e
- (if (:card-error (ex-data e))
- (do
- (log/error e "Pulse card query error")
- (body/render :card-error nil nil nil nil nil))
- (do
- (log/error e "Pulse card render error")
- (body/render :render-error nil nil nil nil nil))))))
+ (let [data (ex-data e)]
+ (cond
+ (:render/too-large? data)
+ (do
+ (log/error "Pulse card query error: results too large")
+ (body/render :card-error/results-too-large nil nil nil nil data))
+
+ (:card-error data)
+ (do
+ (log/error e "Pulse card query error")
+ (body/render :card-error nil nil nil nil nil))
+ :else (do
+ (log/error e "Pulse card render error")
+ (body/render :render-error nil nil nil nil nil)))))))
(mu/defn render-pulse-card :- ::body/RenderedPartCard
"Render a single `card` for a `Pulse` to Hiccup HTML. `result` is the QP results. Returns a map with keys
diff --git a/src/metabase/channel/shared.clj b/src/metabase/channel/shared.clj
index f381682a41d..1bc5a256c8b 100644
--- a/src/metabase/channel/shared.clj
+++ b/src/metabase/channel/shared.clj
@@ -29,10 +29,28 @@
x))
(defn maybe-realize-data-rows
- "Realize the data rows in a [[metabase.notification.payload.execute/Part]]"
+ "Realize the data rows in a [[metabase.notification.payload.execute/Part]].
+
+ If the rows are stored in a StreamingTempFileStorage and the file is too large
+ (> 10MB)[[metabase.notification.settings/notification-temp-file-size-max-bytes]], returns the part with an :error
+ field set so the render pipeline will display an appropriate error message."
[part]
(when part
- (m/update-existing-in part [:result :data :rows] maybe-deref)))
+ (try
+ (m/update-existing-in part [:result :data :rows] maybe-deref)
+ (catch clojure.lang.ExceptionInfo e
+ (if (= :notification/file-too-large
+ (:type (ex-data e)))
+ (let [{:keys [file-size max-size-human-readable]} (ex-data e)
+ file-size-mb (/ file-size 1024.0 1024.0)]
+ (log/warnf "🚫 Result file too large (%.2f MB > %s max). Skipping load to protect memory."
+ file-size-mb max-size-human-readable)
+ ;; Return part with error marker so render pipeline shows an error
+ (update part :result merge {:error (tru "Results too large to display. The query returned too much data to show in this notification.")
+ :render/too-large? true
+ :max-size-human-readable max-size-human-readable}))
+ ;; Re-throw other exceptions
+ (throw e))))))
(defn- schedule-timezone
[]
diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.clj
index 997de63b392..0fdea24eeb7 100644
--- a/src/metabase/driver/sql_jdbc/execute.clj
+++ b/src/metabase/driver/sql_jdbc/execute.clj
@@ -742,7 +742,8 @@
[context :- [:maybe ::lib.schema.info/context]]
(let [download-contexts #{:csv-download :xlsx-download :json-download
:public-csv-download :public-xlsx-download :public-json-download
- :embedded-csv-download :embedded-xlsx-download :embedded-json-download}]
+ :embedded-csv-download :embedded-xlsx-download :embedded-json-download
+ :pulse}]
(boolean (download-contexts context))))
(defn execute-reducible-query
diff --git a/src/metabase/notification/payload/core.clj b/src/metabase/notification/payload/core.clj
index 75d84c5fe11..8363c71c355 100644
--- a/src/metabase/notification/payload/core.clj
+++ b/src/metabase/notification/payload/core.clj
@@ -18,7 +18,7 @@
process-virtual-dashcard]
[notification.payload.temp-storage
cleanup!
- is-cleanable?])
+ cleanable?])
(mr/def ::Notification
"Schema for the notification."
diff --git a/src/metabase/notification/payload/execute.clj b/src/metabase/notification/payload/execute.clj
index c9f22831481..7d0f3631c49 100644
--- a/src/metabase/notification/payload/execute.clj
+++ b/src/metabase/notification/payload/execute.clj
@@ -133,21 +133,6 @@
tag-names)]
(update-in dashcard [:visualization_settings :text] shared.params/substitute-tags tag->param (system/site-locale) (escape-markdown-chars? dashcard))))
-(def ^{:private true
- :doc "If a query has more than the number of rows specified here, we store the data to disk instead of in memory."}
- rows-to-disk-threadhold
- 1000)
-
-(defn- data-rows-to-disk!
- [qp-result context]
- (if (<= (:row_count qp-result) rows-to-disk-threadhold)
- (do
- (log/debugf "Less than %d rows, skip storing %d rows to disk" rows-to-disk-threadhold (:row_count qp-result))
- qp-result)
- (do
- (log/debugf "Storing %d rows to disk" (:row_count qp-result))
- (update-in qp-result [:data :rows] notification.temp-storage/to-temp-file! context))))
-
(defn- fixup-viz-settings
"The viz-settings from :data :viz-settings might be incorrect if there is a cached of the same query.
See #58469.
@@ -155,6 +140,10 @@
[qp-result]
(update-in qp-result [:data :viz-settings] merge (get-in qp-result [:json_query :viz-settings])))
+(def rows-to-disk-threshold
+ "Maximum rows to hold in memory when running notification queries. After this, query results are streamed straight to disk. See [[metabase.notification.payload.temp-storage]] for more details."
+ 2000)
+
(defn execute-dashboard-subscription-card
"Returns subscription result for a card.
@@ -187,7 +176,12 @@
(^:once fn* [query info]
(qp
(qp/userland-query query info)
- nil)))))})
+ ;; Pass streaming rff with 2000 row threshold
+ (notification.temp-storage/notification-rff
+ rows-to-disk-threshold
+ {:dashboard_id dashboard_id
+ :card_id card-id
+ :dashcard_id (u/the-id dashcard)}))))))})
result (result-fn card_id)
series-results (mapv (comp result-fn :id) multi-cards)]
(log/debugf "Dashcard has %d series" (count multi-cards))
@@ -211,11 +205,8 @@
(:card_id dashcard)
(log/with-context {:card_id (:card_id dashcard)}
(let [parameters (merge-default-values parameters)]
- ;; only do this for dashboard subscriptions but not alerts since alerts has only one card, which doesn't eat much
- ;; memory
- ;; TODO: we need to store series result data rows to disk too
+ ;; Streaming to disk is now handled by the query processor rff
(-> (execute-dashboard-subscription-card dashcard parameters)
- (m/update-existing :result data-rows-to-disk! (select-keys dashcard [:dashboard_tab_id :card_id :dashboard_id]))
(m/update-existing :dashcard resolve-inline-parameters parameters))))
(virtual-card-of-type? dashcard "iframe")
@@ -305,9 +296,12 @@
(^:once fn* [query info]
(qp
(qp/userland-query query info)
- nil))))))]
+ ;; Pass streaming rff with 2000 row threshold
+ (notification.temp-storage/notification-rff
+ rows-to-disk-threshold
+ {:card-id card-id})))))))]
(log/debugf "Result has %d rows" (:row_count result))
{:card (t2/select-one :model/Card card-id)
- :result (data-rows-to-disk! result {:card-id card-id})
+ :result result
:type :card}))
diff --git a/src/metabase/notification/payload/temp_storage.clj b/src/metabase/notification/payload/temp_storage.clj
index a8a7264193e..f7c96a6032c 100644
--- a/src/metabase/notification/payload/temp_storage.clj
+++ b/src/metabase/notification/payload/temp_storage.clj
@@ -1,14 +1,28 @@
(ns metabase.notification.payload.temp-storage
- "Util to put data into a temporary file and schedule it for deletion after a specified time period.
+ "Util to put data into a temporary file and delete after the notification sends. Cleanup happens with
+ notification.send/do-after-notification-sent for dashboards and cards which calls cleanup! on each part. This is
+ extended to Object as a no-op and on the type defined here deletes the temporary file.
- Currently used to store card's rows data when sending notification since it can be large and we don't want to keep it in memory."
+ When getting query results for notifications (alerts, subscriptions) once the query row count
+ exceeds [[metabase.notification.payload.execute/rows-to-disk-threshold]], we then start streaming all rows to
+ disk. This ensures that smaller queries don't needlessly write to disk and then reload, while large results don't
+ attempt to reside in memory and kill an instance.
+
+ The key to memory savings here is that the file will not be dereferenced if it is larger than some
+ threshold. Because of this, we are safe to truncate results once the filesize goes above this
+ threshold. See :notification/file-too-large and :notification/truncated?."
(:require
[clojure.java.io :as io]
+ [metabase.analytics.prometheus :as prometheus]
+ [metabase.notification.settings :as notification.settings]
+ [metabase.query-processor.schema :as qp.schema]
+ [metabase.util :as u]
[metabase.util.log :as log]
+ [metabase.util.malli :as mu]
[metabase.util.random :as random]
[taoensso.nippy :as nippy])
(:import
- (java.io File)
+ (java.io BufferedOutputStream DataOutputStream File)
(java.util.concurrent Executors ScheduledThreadPoolExecutor)))
(set! *warn-on-reflection* true)
@@ -30,17 +44,92 @@
(doto (File/createTempFile "notification-" ".npy" @temp-dir)
(.deleteOnExit)))
-(defn- write-to-file!
- [^File file data preamble]
- (nippy/freeze-to-file file {:data data :preamble preamble}))
+(defn- human-readable-size [bytes]
+ (cond (nil? bytes) "0 bytes"
+ (zero? bytes) "0 bytes"
+ :else
+ (let [units ["b" "kb" "mb" "gb" "tb"]
+ magnitude 1024.0]
+ (loop [[unit & remaining] units
+ current (double bytes)]
+ (if (and (seq remaining) (> current magnitude))
+ (recur remaining (/ current magnitude))
+ (format "%.1f %s" current unit))))))
+
+(defn- open-streaming-file!
+ "Create and open a temp file for streaming rows. Returns map with :file and :output-stream."
+ []
+ (let [file (temp-file!)
+ ;; make sure this is the mirror analogue of how we open it in [[read-rows-from-file]]
+ os (-> (io/output-stream file)
+ BufferedOutputStream.
+ DataOutputStream.)]
+ {:file file
+ :output-stream os}))
+
+(defn- write-row-block-to-stream!
+ "Write a collection of rows to the output stream using nippy serialization."
+ [^DataOutputStream os row-block]
+ (nippy/freeze-to-out! os row-block)
+ (.flush os))
-(defn- read-from-file
+(defn- read-rows-from-file
+ "Read rows from a temp file. Returns vector of rows.
+ Throws exception if file is larger than [[notification.settings/notification-temp-file-size-max-bytes]] unless that
+ value is 0. Handles both counted and streaming formats."
[^File file]
- (when (.exists file)
- (let [{:keys [data preamble]} (nippy/thaw-from-file file)]
- (when (seq preamble)
- (log/infof "reading file with preamble: %s" (pr-str preamble)))
- data)))
+ (when-not (.exists file)
+ (throw (ex-info "Temp file no longer exists" {:file file})))
+
+ (let [file-size (.length file)
+ size-human (human-readable-size file-size)]
+ (when (and (pos? (notification.settings/notification-temp-file-size-max-bytes))
+ (> file-size (notification.settings/notification-temp-file-size-max-bytes)))
+ (log/warnf "⚠️ SKIPPING LOAD - File too large: %s (max: %s). File will NOT be loaded into memory."
+ size-human
+ (human-readable-size (notification.settings/notification-temp-file-size-max-bytes)))
+ (throw (ex-info "Result file too large to load into memory"
+ {:type :notification/file-too-large
+ :file-size file-size
+ :max-size (notification.settings/notification-temp-file-size-max-bytes)
+ :max-size-human-readable (human-readable-size
+ (notification.settings/notification-temp-file-size-max-bytes))})))
+
+ (log/infof "πŸ“‚ Loading streamed results from disk: %s" size-human)
+
+ (let [start-time (System/nanoTime)
+ result (with-open [is (-> (io/input-stream file)
+ java.io.BufferedInputStream.
+ java.io.DataInputStream.)]
+ ;; Read preamble
+ (let [{:keys [preamble]} (nippy/thaw-from-in! is)]
+ (when (seq preamble)
+ (log/debugf "File context: %s" (pr-str preamble))))
+
+ ;; Read row count/marker
+ (let [count-or-marker (nippy/thaw-from-in! is)]
+ (if (= count-or-marker ::streaming)
+ ;; Streaming format - read until EOF
+ (loop [rows (transient [])]
+ (let [row-block (try
+ (nippy/thaw-from-in! is)
+ (catch java.io.EOFException _
+ ::eof))]
+ (if (= row-block ::eof)
+ (persistent! rows)
+ (recur (reduce conj! rows row-block)))))
+
+ ;; Counted format - read exact number of rows
+ (loop [rows (transient [])
+ i 0]
+ (if (< i count-or-marker)
+ (recur (conj! rows (nippy/thaw-from-in! is))
+ (inc i))
+ (persistent! rows))))))
+ elapsed-ms (/ (- (System/nanoTime) start-time) 1000000.0)
+ row-count (count result)]
+ (log/infof "βœ… Loaded %d rows from disk in %.0f ms (%s)" row-count elapsed-ms size-human)
+ result)))
(.addShutdownHook
(Runtime/getRuntime)
@@ -56,19 +145,7 @@
Object
(cleanup! [_] nil))
-(defn- human-readable-size [bytes]
- (cond (nil? bytes) "0 bytes"
- (zero? bytes) "0 bytes"
- :else
- (let [units ["b" "kb" "mb" "gb" "tb"]
- magnitude 1024.0]
- (loop [[unit & remaining] units
- current (double bytes)]
- (if (and (seq remaining) (> current magnitude))
- (recur remaining (/ current magnitude))
- (format "%.1f %s" current unit))))))
-
-(deftype TempFileStorage [^File file]
+(deftype StreamingTempFileStorage [^File file context]
Cleanable
(cleanup! [_]
(when (.exists file)
@@ -76,44 +153,176 @@
clojure.lang.IDeref
(deref [_]
- (if (.exists file)
- (do
- (log/infof "reading temp storage file of size: %s" (human-readable-size (.length file)))
- (read-from-file file))
- (throw (ex-info "File no longer exists" {:file file}))))
+ (read-rows-from-file file))
+
+ clojure.lang.IPending
+ (isRealized [_]
+ ;; Always return false so REPL tools like fipp won't auto-deref
+ ;; Reading from disk is expensive and should be explicit
+ false)
Object
(toString [_]
- (str "#TempFileStorage{:file " file "}"))
+ (if (.exists file)
+ (format "#StreamingTempFileStorage{:file %s, :size %s, :context %s}"
+ (.getName file)
+ (human-readable-size (.length file))
+ (pr-str context))
+ (format "#StreamingTempFileStorage{:file %s (deleted), :context %s}"
+ (.getName file)
+ (pr-str context))))
- ;; Add equality behavior
(equals [_this other]
- (and (instance? TempFileStorage other)
- (= file (.file ^TempFileStorage other))))
+ (and (instance? StreamingTempFileStorage other)
+ (= file (.file ^StreamingTempFileStorage other))))
(hashCode [_]
(.hashCode file)))
+(defmethod print-method StreamingTempFileStorage
+ [^StreamingTempFileStorage storage ^java.io.Writer w]
+ (.write w (.toString storage)))
+
+(mu/defn notification-rff :- ::qp.schema/rff
+ "Reducing function factory for notifications that streams to disk when threshold exceeded.
+
+ Returns an rff (function that takes metadata and returns an rf) that:
+ - Accumulates rows in memory initially
+ - Once max-row-count is reached, switches to streaming mode
+ - In streaming mode, writes all accumulated rows to disk then streams subsequent rows
+ - Final result contains either in-memory rows or a StreamingTempFileStorage reference
+
+ Parameters:
+ - max-row-count: Maximum number of rows to keep in memory before streaming to disk
+ - context: Optional context map to include in temp file preamble (for debugging)"
+ ([max-row-count]
+ (notification-rff max-row-count {}))
+ ([max-row-count context]
+ (fn rff [metadata]
+ (let [row-count (volatile! 0)
+ rows (volatile! (transient []))
+ streaming? (volatile! false)
+ streaming-state (volatile! nil)] ; {:file File :output-stream DataOutputStream}
+ (fn notification-rf
+ ;; Init arity
+ ([]
+ {:data metadata})
+
+ ;; Completion arity
+ ([result]
+ {:pre [(map? (unreduced result))]}
+ (let [result (unreduced result)]
+ (if @streaming?
+ ;; Close the streaming file and return TempFileStorage
+ (let [{:keys [^DataOutputStream output-stream ^File file]} @streaming-state]
+ (try
+ (when-let [remaining-rows (seq (persistent! @rows))]
+ (write-row-block-to-stream! output-stream remaining-rows))
+ (.close output-stream)
+ (let [file-size (.length file)
+ max-bytes (notification.settings/notification-temp-file-size-max-bytes)]
+ (prometheus/inc! :metabase-notification/temp-storage
+ {:storage (cond (:notification/truncated? @streaming-state) :truncated
+ (zero? max-bytes) :not-limited
+ (< max-bytes file-size) :above-threshold
+ :else :disk)})
+ (log/infof "πŸ’Ύ Stored %d rows to disk: %s (never loaded into memory)%s"
+ @row-count
+ (human-readable-size file-size)
+ (if (:notification/truncated? @streaming-state)
+ " (note query results were truncated)"
+ ""))
+ (-> result
+ (assoc :row_count @row-count
+ :status :completed
+ :data.rows-file-size file-size)
+ (cond-> (:notification/truncated? @streaming-state)
+ (assoc :notification/truncated? true))
+ (assoc-in [:data :rows] (StreamingTempFileStorage. file context))))
+ (catch Exception e
+ (u/ignore-exceptions (.close output-stream))
+ (throw e))))
+
+ ;; Return in-memory rows
+ (do
+ (prometheus/inc! :metabase-notification/temp-storage
+ {:storage :memory})
+ (log/infof "βœ“ Completed with %d rows in memory (under threshold)" @row-count)
+ (-> result
+ (assoc :row_count @row-count
+ :status :completed)
+ (assoc-in [:data :rows] (persistent! @rows)))))))
+
+ ;; Step arity - accumulate rows
+ ([result row]
+ ;; unconditionally increment row count and add rows to internal volatile transient collector. If we are
+ ;; streaming to disk, we periodically flush this to disk; if we are collecting in memory the collection is
+ ;; used the same as in the default-rff
+ (vswap! row-count inc)
+ (vswap! rows conj! row)
+ (if @streaming?
+ ;; Already streaming - write row directly to file
+ (let [{:keys [^DataOutputStream output-stream ^File file]} @streaming-state]
+ (if-not (zero? (mod @row-count 5000))
+ ;; only flush the volatile transient every 5000 rows. If we aren't on a 5,000 row multiple, we've
+ ;; already added the row above so nothing to do here.
+ result
+ (do
+ ;; else we write the row block and then check if the size has gotten too large and we can abandon
+ ;; the query with `(reduced result)`
+ (write-row-block-to-stream! output-stream (persistent! @rows))
+ (vreset! rows (transient []))
+ (if (and (pos? (notification.settings/notification-temp-file-size-max-bytes))
+ (> (.length file) (* 1.3 (notification.settings/notification-temp-file-size-max-bytes))))
+ (do (vswap! streaming-state assoc :notification/truncated? true)
+ (log/warnf "Results have exceeded 1.3 times of `notification-temp-file-size-max-bytes` of %s (max: %s). Truncating query results. %s"
+ (human-readable-size (.length file))
+ (human-readable-size (notification.settings/notification-temp-file-size-max-bytes))
+ (when context (format "(%s)" (pr-str context))))
+ (reduced result))
+ result))))
+
+ ;; Still in memory - check if we should start streaming
+ (do
+ ;; Check if we've hit the threshold to switch to disk based streaming
+ (when (>= @row-count max-row-count)
+ (log/infof "Row count reached threshold (%d), switching to streaming mode"
+ max-row-count)
+ ;; Open streaming file
+ (let [{:keys [file ^DataOutputStream output-stream]} (open-streaming-file!)]
+ (try
+ ;; Write preamble
+ (nippy/freeze-to-out! output-stream {:preamble context})
+ (.flush output-stream)
+
+ ;; Write sentinel indicating streaming format
+ (nippy/freeze-to-out! output-stream ::streaming)
+ (.flush output-stream)
+
+ ;; Write all accumulated rows
+ (write-row-block-to-stream! output-stream (persistent! @rows))
+
+ ;; Switch to streaming mode
+ (vreset! streaming? true)
+ (vreset! rows (transient [])) ; Clear memory
+ (vreset! streaming-state {:file file :output-stream output-stream})
+
+ (catch Exception e
+ (u/ignore-exceptions (.close output-stream))
+ (u/ignore-exceptions (io/delete-file file))
+ (throw e)))))
+ result))))))))
+
;; ------------------------------------------------------------------------------------------------;;
;; Public APIs ;;
;; ------------------------------------------------------------------------------------------------;;
-(defn is-cleanable?
+(defn cleanable?
"Returns true if x implements the Cleanable protocol"
[x]
(satisfies? Cleanable x))
-(defn to-temp-file!
- "Write data to a temporary file. Returns a TempFileStorage type that:
- - Implements IDeref - use @ to read the data from the file
- - Implements Cleanable - call cleanup! when the file is no longer needed
-
- You may include an optional `preamble` which will be logged when the file is read. Helpful to leave a breadcrumb of
- which dashboard/card the file might be from. "
- ^TempFileStorage
- ([data] (to-temp-file! data {}))
- ([data preamble]
- (let [f (temp-file!)]
- (write-to-file! f data preamble)
- (log/debug "stored data in temp file" {:length (.length ^File f)})
- (TempFileStorage. f))))
+(defn streaming-temp-file?
+ "Check if x is a StreamingTempFileStorage instance."
+ [x]
+ (instance? StreamingTempFileStorage x))
diff --git a/src/metabase/notification/settings.clj b/src/metabase/notification/settings.clj
index 5dc99aa44dd..b2e80479873 100644
--- a/src/metabase/notification/settings.clj
+++ b/src/metabase/notification/settings.clj
@@ -18,3 +18,13 @@
:export? false
:type :integer
:visibility :internal)
+
+(defsetting notification-temp-file-size-max-bytes
+ "The maximum file size that will be created when storing notification query results on disk.
+ Note this is in BYTES. Default value is 10485760 which is `10 * 1024 * 1024`. To disable this size limit set the
+ value to 0."
+ :type :integer
+ :default (* 10 1024 1024)
+ :export? false
+ :setter :none
+ :visibility :internal)
diff --git a/src/metabase/query_processor/pipeline.clj b/src/metabase/query_processor/pipeline.clj
index 7c35abee9a0..06829405d87 100644
--- a/src/metabase/query_processor/pipeline.clj
+++ b/src/metabase/query_processor/pipeline.clj
@@ -1,6 +1,7 @@
(ns metabase.query-processor.pipeline
(:require
[clojure.core.async :as a]
+ [metabase.analytics.core :as analytics]
[metabase.driver :as driver]
[metabase.query-processor.error-type :as qp.error-type]
[metabase.util.i18n :as i18n]
@@ -79,6 +80,8 @@
(fn wrapper
([] (rf))
([acc]
+ (analytics/inc! :metabase-query-processor/query
+ {:driver driver/*driver* :status "success"})
(some-> *canceled-chan* a/close!)
(rf acc))
([acc row]
diff --git a/src/metabase/query_processor/reducible.clj b/src/metabase/query_processor/reducible.clj
index d0c9fbd08b6..12c41d926c7 100644
--- a/src/metabase/query_processor/reducible.clj
+++ b/src/metabase/query_processor/reducible.clj
@@ -1,8 +1,6 @@
(ns metabase.query-processor.reducible
(:require
[clojure.core.async :as a]
- [metabase.analytics.core :as analytics]
- [metabase.driver :as driver]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.query-processor.pipeline :as qp.pipeline]
[metabase.query-processor.schema :as qp.schema]
@@ -26,7 +24,6 @@
([result]
{:pre [(map? (unreduced result))]}
;; if the result is a clojure.lang.Reduced, unwrap it so we always get back the standard-format map
- (analytics/inc! :metabase-query-processor/query {:driver driver/*driver* :status "success"})
(-> (unreduced result)
(assoc :row_count @row-count
:status :completed)
diff --git a/src/metabase/query_processor/streaming.clj b/src/metabase/query_processor/streaming.clj
index 65325fa1eb5..c17a1efc257 100644
--- a/src/metabase/query_processor/streaming.clj
+++ b/src/metabase/query_processor/streaming.clj
@@ -2,8 +2,6 @@
(:refer-clojure :exclude [every? some])
(:require
[clojure.string :as str]
- [metabase.analytics.core :as analytics]
- [metabase.driver :as driver]
;; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use
;; Lib soon
^{:clj-kondo/ignore [:discouraged-namespace]}
@@ -180,7 +178,6 @@
{:data initial-metadata})
([result]
- (analytics/inc! :metabase-query-processor/query {:driver driver/*driver* :status "success"})
(assoc result
:row_count @row-count
:status :completed))
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index 61689668b4f..448b86e8680 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -30,6 +30,7 @@
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.metadata-providers.mock :as providers.mock]
+ [metabase.notification.payload.temp-storage :as temp-storage]
[metabase.query-processor :as qp]
[metabase.query-processor.compile :as qp.compile]
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
@@ -1896,3 +1897,57 @@
(doseq [[provider host] tests]
(let [database {:details {:host host} :engine :postgres}]
(is (= provider (provider-detection/detect-provider-from-database database))))))))
+
+(deftest ^:parallel complex-types-in-notification-payload
+ (mt/test-driver :postgres
+ (testing "we handle complex types in notifications"
+ (let [sql "SELECT
+ i AS id,
+ 'User_' || i AS username,
+ -- JSON types
+ ('{\"userId\": ' || i || ', \"score\": ' || (i % 1000) || ', \"active\": ' || (i % 2 = 0)::text || '}')::jsonb AS settings,
+ -- Arrays
+ ARRAY['tag' || (i % 10), 'category' || (i % 5)]::text[] AS tags,
+ ARRAY[i, i*2, i*3]::integer[] AS numbers,
+ -- UUID
+ '7e3cd49d-bfe1-4620-83dd-0c163719175c'::uuid AS uuid,
+ -- Network
+ ('192.168.' || (i % 255) || '.' || ((i*7) % 255))::inet AS ip,
+ -- Geometric
+ point(i % 180 - 90, i % 360 - 180) AS coordinates,
+ circle(point(i % 100, i % 100), 50) AS area,
+ -- Text search
+ to_tsvector('english', 'Content for row ' || i) AS search_data,
+ -- Binary
+ decode(md5(i::text), 'hex') AS hash,
+ -- Range types
+ int4range(i, i + 100) AS value_range,
+ -- Money
+ ((RANDOM() * 1000)::numeric(10,2))::money AS price,
+ -- Standard
+ NOW() - ((i % 365) || ' days')::interval AS created_at
+ FROM generate_series(1, 5000) AS i;"
+ results (qp/process-query (mt/native-query {:query sql})
+ (temp-storage/notification-rff 500))]
+ (is (integer? (:data.rows-file-size results)))
+ (is (temp-storage/streaming-temp-file? (-> results :data :rows)))
+ (is (=? [1
+ "User_1"
+ "{\"score\": 1, \"active\": false, \"userId\": 1}"
+ ["tag1" "category1"]
+ [1 2 3]
+ ;; match on type
+ uuid?
+ "192.168.1.7"
+ "(-89.0,-179.0)"
+ "<(1.0,1.0),50.0>"
+ "'1':4 'content':1 'row':3"
+ ;; match on type
+ bytes?
+ "[1,101)"
+ ;; match on type
+ decimal?
+ ;; we don't have an easy way to recognize a datetime string. assuming if it's a string and query
+ ;; doesnt error then it worked
+ string?]
+ (-> results :data :rows deref first)))))))
diff --git a/test/metabase/notification/payload/impl/card_test.clj b/test/metabase/notification/payload/impl/card_test.clj
index aba8e977adf..52ff4fc5ee1 100644
--- a/test/metabase/notification/payload/impl/card_test.clj
+++ b/test/metabase/notification/payload/impl/card_test.clj
@@ -137,8 +137,8 @@
(deftest card-with-rows-saved-to-disk-test
(testing "whether the rows of a card saved to disk or in memory, all channels should work\n"
(doseq [limit [1 10]]
- (with-redefs [notification.payload.execute/rows-to-disk-threadhold 5]
- (testing (if (> limit @#'notification.payload.execute/rows-to-disk-threadhold)
+ (with-redefs [notification.payload.execute/rows-to-disk-threshold 5]
+ (testing (if (> limit @#'notification.payload.execute/rows-to-disk-threshold)
"card has rows saved to disk"
"card has rows saved in memory")
(notification.tu/with-notification-testing-setup!
@@ -184,7 +184,7 @@
(deftest cards-with-rows-saved-to-disk-will-cleanup-the-files
(let [f (atom nil)
orig-execute-fn @#'notification.payload.execute/execute-card]
- (with-redefs [notification.payload.execute/rows-to-disk-threadhold 1
+ (with-redefs [notification.payload.execute/rows-to-disk-threshold 1
notification.payload.execute/execute-card
(fn [& args]
(let [result (apply orig-execute-fn args)]
@@ -197,9 +197,9 @@
:handlers [@notification.tu/default-email-handler]}]
(notification/send-notification! notification)
(testing "sanity check that the file exists in the first place"
- (is (notification.payload/is-cleanable? @f)))
+ (is (notification.payload/cleanable? @f)))
(testing "the files are cleaned up"
- (is (not (.exists ^java.io.File (.file ^metabase.notification.payload.temp_storage.TempFileStorage @f))))))))))
+ (is (not (.exists ^java.io.File (.file ^metabase.notification.payload.temp_storage.StreamingTempFileStorage @f))))))))))
(deftest ensure-constraints-test
(testing "Validate card queries are limited by `default-query-constraints`"
diff --git a/test/metabase/notification/payload/temp_storage_test.clj b/test/metabase/notification/payload/temp_storage_test.clj
index 271d1748f7b..de7cf56ba45 100644
--- a/test/metabase/notification/payload/temp_storage_test.clj
+++ b/test/metabase/notification/payload/temp_storage_test.clj
@@ -1,47 +1,153 @@
(ns metabase.notification.payload.temp-storage-test
(:require
[clojure.test :refer :all]
- [metabase.notification.payload.temp-storage :as temp-storage]))
+ [metabase.notification.payload.temp-storage :as temp-storage]
+ [metabase.test :as mt]))
(set! *warn-on-reflection* true)
-(deftest basic-write-read-test
- (testing "basic write and read operations"
- (let [test-data {:a 1 :b "test" :c [1 2 3]}
- storage (temp-storage/to-temp-file! test-data)]
- (try
- (is (= test-data @storage))
- (finally
- (temp-storage/cleanup! storage))))))
-
-(deftest cleanup-test
- (testing "file is actually deleted after cleanup"
- (let [storage (temp-storage/to-temp-file! "test-data")
- ^java.io.File file (.file storage)]
- (is (.exists file))
+(defn- run-rff
+ "Helper to run notification-rff with given max-rows and row data."
+ [max-rows row-data]
+ (let [metadata {:cols [{:name "id"} {:name "value"}]}
+ rff (temp-storage/notification-rff max-rows {})
+ rf (rff metadata)]
+ (transduce identity rf row-data)))
+
+(defn- rows
+ "Generate n rows of test data."
+ [n]
+ (for [i (range n)] [i (* i 10)]))
+
+(deftest streaming-threshold-test
+ (testing "Under threshold - rows stay in memory"
+ (let [result (run-rff 5 (rows 4))]
+ (is (= 4 (:row_count result)))
+ (is (= :completed (:status result)))
+ (is (vector? (get-in result [:data :rows])))
+ (is (= (rows 4) (get-in result [:data :rows])))))
+
+ (testing "At threshold - rows stream to disk"
+ (let [result (run-rff 5 (rows 5))
+ storage (get-in result [:data :rows])]
+ (is (= 5 (:row_count result)))
+ (is (temp-storage/streaming-temp-file? storage))
+ (is (= (rows 5) @storage))
+ (temp-storage/cleanup! storage)))
+
+ (testing "Over threshold - rows stream to disk"
+ (let [result (run-rff 5 (rows 6))
+ storage (get-in result [:data :rows])]
+ (is (= 6 (:row_count result)))
+ (is (temp-storage/streaming-temp-file? storage))
+ (is (= (rows 6) @storage))
+ (temp-storage/cleanup! storage)))
+
+ (testing "Over max-file-size threshold, aborts query"
+ (mt/with-temporary-setting-values [notification-temp-file-size-max-bytes (* 4 1024)]
+ (let [many-rows 5000000
+ result (run-rff 5 (rows many-rows))
+ storage (get-in result [:data :rows])]
+ ;; row count here is how many query results were returned
+ (is (< (:row_count result) many-rows))
+ (is (temp-storage/streaming-temp-file? storage))
+ (is (:notification/truncated? result))
+ (mt/with-temporary-setting-values [notification-temp-file-size-max-bytes 0]
+ ;; remove enforcing notification size limit before dereferencing and then see that we _stored_ fewer
+ ;; results than otherwise might have been returned. Without resetting this, dereferencing the rows would
+ ;; error as it was larger than the file size limit
+
+ (is (< (count @storage) many-rows))))))
+ ;; in testing on my machine, this test takes this whole test from 262ms to 1550 ms. Seems worthwhile to me
+ (testing "When bytes size is set to 0 allows for arbitrary sizes"
+ (mt/with-temporary-setting-values [notification-temp-file-size-max-bytes 0]
+ (let [many-rows 2500000
+ result (run-rff 5 (rows many-rows))
+ storage (get-in result [:data :rows])]
+ ;; row count here is how many query results were returned
+ (is (= (:row_count result) many-rows))
+ (is (temp-storage/streaming-temp-file? storage))
+ ;; on my machine this was 28.5mb. Don't need to push this into GB but demonstrate we are above the 10mb limit
+ (is (and (:data.rows-file-size result)
+ (> (:data.rows-file-size result) (* 10 1024 1024)))
+ (format "data.rows-file-size was only : %d" (:data.rows-file-size result)))
+ (is (not (:notification/truncated? result)))
+ (is (= (count @storage) many-rows))))))
+
+(deftest streaming-edge-cases-test
+ (testing "Empty rows"
+ (let [result (run-rff 5 [])]
+ (is (= 0 (:row_count result)))
+ (is (vector? (get-in result [:data :rows])))
+ (is (empty? (get-in result [:data :rows])))))
+
+ (testing "Single row"
+ (let [result (run-rff 5 [[42]])]
+ (is (= 1 (:row_count result)))
+ (is (= [[42]] (get-in result [:data :rows])))))
+
+ (testing "Large row count"
+ (let [result (run-rff 100 (for [i (range 150)] [i]))
+ storage (get-in result [:data :rows])]
+ (is (= 150 (:row_count result)))
+ (is (temp-storage/streaming-temp-file? storage))
+ (is (= 150 (count @storage)))
+ (is (= [0] (first @storage)))
+ (is (= [149] (last @storage)))
+ (temp-storage/cleanup! storage))))
+
+(deftest streaming-data-integrity-test
+ (testing "Repeated values preserve correctly"
+ (let [metadata {:cols (for [c ["a" "b" "c" "d" "e"]] {:name c})}
+ rff (temp-storage/notification-rff 5 {})
+ rf (rff metadata)
+ data (for [i (range 6)] (vec (repeat 5 i)))
+ result (transduce identity rf data)
+ storage (get-in result [:data :rows])]
+ (is (temp-storage/streaming-temp-file? storage))
+ (is (= [[0 0 0 0 0]
+ [1 1 1 1 1]
+ [2 2 2 2 2]
+ [3 3 3 3 3]
+ [4 4 4 4 4]
+ [5 5 5 5 5]]
+ @storage))
+ (temp-storage/cleanup! storage))))
+
+(deftest streaming-file-size-test
+ (testing "File size is included in result"
+ (let [result (run-rff 5 (rows 10))
+ storage (get-in result [:data :rows])]
+ (is (number? (:data.rows-file-size result)))
+ (is (pos? (:data.rows-file-size result)))
+ (is (> (:data.rows-file-size result) 50))
+ (temp-storage/cleanup! storage))))
+
+(deftest streaming-cleanup-test
+ (testing "StreamingTempFileStorage can be cleaned up"
+ (let [result (run-rff 5 (rows 10))
+ storage (get-in result [:data :rows])]
+ (is (= 10 (count @storage)))
(temp-storage/cleanup! storage)
- (is (not (.exists file)))
- (is (thrown-with-msg?
- Exception
- #"File no longer exists"
- @storage)))))
-
-(deftest nil-handling-test
- (testing "can handle nil values"
- (let [storage (temp-storage/to-temp-file! nil)]
- (try
- (is (nil? @storage))
- (finally
- (temp-storage/cleanup! storage))))))
-
-(deftest equality-test
- (testing "equality behavior"
- (let [data {:test "data"}
- storage1 (temp-storage/to-temp-file! data)
- storage2 (temp-storage/to-temp-file! data)]
- (try
- (is (= storage1 storage1))
- (is (not= storage1 storage2))
- (finally
- (temp-storage/cleanup! storage1)
- (temp-storage/cleanup! storage2))))))
+ (is (thrown? Exception @storage)))))
+
+(deftest streaming-pending-test
+ (testing "IPending returns false to prevent auto-deref"
+ (let [result (run-rff 5 (rows 10))
+ storage (get-in result [:data :rows])]
+ (is (not (realized? storage)))
+ (temp-storage/cleanup! storage))))
+
+(deftest fidelity
+ (testing "Untruncated results should be equal"
+ (let [query (mt/mbql-query orders)]
+ ;; its the default value but just ensuring it is high enough
+ (mt/with-temporary-setting-values [notification-temp-file-size-max-bytes (* 10 1024 1024)]
+ (let [qp-results (mt/process-query query)
+ temp-file-results (mt/process-query query
+ (temp-storage/notification-rff 2000))]
+ (is (not (:notification/truncated? temp-file-results)))
+ (is (temp-storage/streaming-temp-file? (-> temp-file-results :data :rows)))
+ (is (= (-> qp-results :data :rows)
+ (-> temp-file-results :data :rows deref)))
+ (-> temp-file-results :data :rows temp-storage/cleanup!))))))
diff --git a/test/metabase/pulse/dashboard_subscription_test.clj b/test/metabase/pulse/dashboard_subscription_test.clj
index 27a1c91b20b..78f58368e30 100644
--- a/test/metabase/pulse/dashboard_subscription_test.clj
+++ b/test/metabase/pulse/dashboard_subscription_test.clj
@@ -1287,7 +1287,7 @@
(str "metabase-test" (random/random-name)))]
(.mkdirs dir)
dir))
- notification.payload.execute/rows-to-disk-threadhold 1
+ notification.payload.execute/rows-to-disk-threshold 1
channel/send! (fn [& _args]
(testing "sanity check that there are files there to cleanup"
(is (not-empty (.listFiles ^java.io.File @@#'notification.temp-storage/temp-dir)))))]
@@ -1302,9 +1302,9 @@
(deftest dashboard-with-rows-saved-to-disk-test
(testing "whether the rows of a dashboard saved to disk or in memory, all channels should work"
(doseq [limit [1 #_10]]
- (with-redefs [notification.payload.execute/rows-to-disk-threadhold 5]
+ (with-redefs [notification.payload.execute/rows-to-disk-threshold 5]
(notification.tu/with-channel-fixtures [:channel/slack]
- (testing (if (> limit @#'notification.payload.execute/rows-to-disk-threadhold)
+ (testing (if (> limit @#'notification.payload.execute/rows-to-disk-threshold)
"dashboard has rows saved to disk"
"dashboard has rows saved in memory")
(mt/with-temp [:model/Card {card-id :id} {:name pulse.test-util/card-name
diff --git a/test/metabase/pulse/render/test_util.clj b/test/metabase/pulse/render/test_util.clj
index 7fa12b51f80..b8ef638735d 100644
--- a/test/metabase/pulse/render/test_util.clj
+++ b/test/metabase/pulse/render/test_util.clj
@@ -12,6 +12,7 @@
[metabase.channel.render.core :as channel.render]
[metabase.channel.render.image-bundle :as image-bundle]
[metabase.channel.render.js.svg :as js.svg]
+ [metabase.channel.shared :as channel.shared]
[metabase.notification.payload.execute :as notification.execute]
[metabase.query-processor :as qp]
[metabase.query-processor.card :as qp.card]
@@ -182,7 +183,8 @@
([dashcard-id parameters]
(let [dashcard (t2/select-one :model/DashboardCard :id dashcard-id)
card (t2/select-one :model/Card :id (:card_id dashcard))
- {:keys [result dashcard]} (notification.execute/execute-dashboard-subscription-card dashcard parameters)]
+ {:keys [result dashcard]} (channel.shared/maybe-realize-data-rows
+ (notification.execute/execute-dashboard-subscription-card dashcard parameters))]
(with-redefs [js.svg/svg-string->bytes identity
image-bundle/make-image-bundle (fn [_ s]
{:image-src s