The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
use disk at query time of notification payload creation (#64390)
dpsuttondb11feef9b73
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex ec5bdc350da..5fcbc248c8c 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -364,6 +364,10 @@(prometheus/counter :metabase-notification/send-error{:description "Number of errors when sending notifications.":labels [:payload-type]})+ (prometheus/counter :metabase-notification/temp-storage+ {:description "Number and type of temporary storage uses"+ ;; memory, disk, above-threshold, truncated, not-limited+ :labels [:storage]})(prometheus/histogram :metabase-notification/wait-duration-ms{:description "Duration in milliseconds that notifications wait in the processing queue before being picked up for delivery.":labels [:payload-type]diff --git a/src/metabase/channel/email/result_attachment.clj b/src/metabase/channel/email/result_attachment.cljindex f5343408fe1..58d515be2e6 100644--- a/src/metabase/channel/email/result_attachment.clj+++ b/src/metabase/channel/email/result_attachment.clj@@ -89,21 +89,23 @@dashcard :dashcardresult :result:as part}]- (when (pos-int? (:row_count result))+ (when (and (or (:include_csv card) (:include_xls card))+ (pos-int? (:row_count result)))(let [maybe-realize-data-rows (requiring-resolve 'metabase.channel.shared/maybe-realize-data-rows)result (:result (maybe-realize-data-rows part))visualizer-title (when (and dashcard (get-in dashcard [:visualization_settings :visualization]))(not-empty (get-in dashcard [:visualization_settings :visualization :settings :card.title])))filename-prefix (or visualizer-title original-card-name)]- (->>- [(when-let [temp-file (and (:include_csv card)- (create-temp-file-or-throw! "csv"))]- (with-open [os (io/output-stream temp-file)]- (stream-api-results-to-export-format! os {:export-format :csv :format-rows? format-rows :pivot? pivot-results} result))- (create-result-attachment-map "csv" filename-prefix temp-file))- (when-let [temp-file (and (:include_xls card)- (create-temp-file-or-throw! "xlsx"))]- (with-open [os (io/output-stream temp-file)]- (stream-api-results-to-export-format! os {:export-format :xlsx :format-rows? format-rows :pivot? pivot-results} result))- (create-result-attachment-map "xlsx" filename-prefix temp-file))]- (filterv some?)))))+ (when-not (:render/too-large? result)+ (->>+ [(when-let [temp-file (and (:include_csv card)+ (create-temp-file-or-throw! "csv"))]+ (with-open [os (io/output-stream temp-file)]+ (stream-api-results-to-export-format! os {:export-format :csv :format-rows? format-rows :pivot? pivot-results} result))+ (create-result-attachment-map "csv" filename-prefix temp-file))+ (when-let [temp-file (and (:include_xls card)+ (create-temp-file-or-throw! "xlsx"))]+ (with-open [os (io/output-stream temp-file)]+ (stream-api-results-to-export-format! os {:export-format :xlsx :format-rows? format-rows :pivot? pivot-results} result))+ (create-result-attachment-map "xlsx" filename-prefix temp-file))]+ (filterv some?))))))diff --git a/src/metabase/channel/render/body.clj b/src/metabase/channel/render/body.cljindex 54a20107794..8f66f0ca710 100644--- a/src/metabase/channel/render/body.clj+++ b/src/metabase/channel/render/body.clj@@ -639,6 +639,21 @@[_chart-type _render-type _timezone-id _card _dashcard _data]@card-error-rendered-info)+(mu/defmethod render :card-error/results-too-large :- ::RenderedPartCard+ [_chart-type _render-type _timezone-id _card _dashcard data]+ (let [filesize-limit (:max-size-human-readable data "10.0 mb")]+ {:attachments nil,+ :content [:div+ {:style "font-family: Lato, \"Helvetica Neue\", Helvetica, Arial, sans-serif; margin: 0px 0;"}+ [:div+ {:style "background-color: #F9FBFC; border: 1px solid #DCE1E4; border-radius: 8px; padding: 16px;"}+ [:p+ {:style "margin: 4px; font-size: 14px; font-weight: 700; color: #4C5773;"}+ (trs "This chart exceeded the {0} size limit." filesize-limit)]+ [:p+ {:style "margin: 4px; font-size: 12px; font-weight: 400; color: #696E7B;"}+ (trs "You can make it smaller by adding filters, or summarizing the data.")]]]}))+(mu/defmethod render :render-error :- ::RenderedPartCard[_chart-type _render-type _timezone-id _card _dashcard _data]@error-rendered-info)diff --git a/src/metabase/channel/render/card.clj b/src/metabase/channel/render/card.cljindex 12d6aba8d11..c34bc9ae6a2 100644--- a/src/metabase/channel/render/card.clj+++ b/src/metabase/channel/render/card.clj@@ -153,13 +153,20 @@(log/debugf "Rendering pulse card with chart-type %s and render-type %s" chart-type render-type)(body/render chart-type render-type timezone-id card dashcard data))(catch Throwable e- (if (:card-error (ex-data e))- (do- (log/error e "Pulse card query error")- (body/render :card-error nil nil nil nil nil))- (do- (log/error e "Pulse card render error")- (body/render :render-error nil nil nil nil nil))))))+ (let [data (ex-data e)]+ (cond+ (:render/too-large? data)+ (do+ (log/error "Pulse card query error: results too large")+ (body/render :card-error/results-too-large nil nil nil nil data))++ (:card-error data)+ (do+ (log/error e "Pulse card query error")+ (body/render :card-error nil nil nil nil nil))+ :else (do+ (log/error e "Pulse card render error")+ (body/render :render-error nil nil nil nil nil)))))))(mu/defn render-pulse-card :- ::body/RenderedPartCard"Render a single `card` for a `Pulse` to Hiccup HTML. `result` is the QP results. Returns a map with keysdiff --git a/src/metabase/channel/shared.clj b/src/metabase/channel/shared.cljindex f381682a41d..1bc5a256c8b 100644--- a/src/metabase/channel/shared.clj+++ b/src/metabase/channel/shared.clj@@ -29,10 +29,28 @@x))(defn maybe-realize-data-rows- "Realize the data rows in a [[metabase.notification.payload.execute/Part]]"+ "Realize the data rows in a [[metabase.notification.payload.execute/Part]].++ If the rows are stored in a StreamingTempFileStorage and the file is too large+ (> 10MB)[[metabase.notification.settings/notification-temp-file-size-max-bytes]], returns the part with an :error+ field set so the render pipeline will display an appropriate error message."[part](when part- (m/update-existing-in part [:result :data :rows] maybe-deref)))+ (try+ (m/update-existing-in part [:result :data :rows] maybe-deref)+ (catch clojure.lang.ExceptionInfo e+ (if (= :notification/file-too-large+ (:type (ex-data e)))+ (let [{:keys [file-size max-size-human-readable]} (ex-data e)+ file-size-mb (/ file-size 1024.0 1024.0)]+ (log/warnf "π« Result file too large (%.2f MB > %s max). Skipping load to protect memory."+ file-size-mb max-size-human-readable)+ ;; Return part with error marker so render pipeline shows an error+ (update part :result merge {:error (tru "Results too large to display. The query returned too much data to show in this notification.")+ :render/too-large? true+ :max-size-human-readable max-size-human-readable}))+ ;; Re-throw other exceptions+ (throw e))))))(defn- schedule-timezone[]diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.cljindex 997de63b392..0fdea24eeb7 100644--- a/src/metabase/driver/sql_jdbc/execute.clj+++ b/src/metabase/driver/sql_jdbc/execute.clj@@ -742,7 +742,8 @@[context :- [:maybe ::lib.schema.info/context]](let [download-contexts #{:csv-download :xlsx-download :json-download:public-csv-download :public-xlsx-download :public-json-download- :embedded-csv-download :embedded-xlsx-download :embedded-json-download}]+ :embedded-csv-download :embedded-xlsx-download :embedded-json-download+ :pulse}](boolean (download-contexts context))))(defn execute-reducible-querydiff --git a/src/metabase/notification/payload/core.clj b/src/metabase/notification/payload/core.cljindex 75d84c5fe11..8363c71c355 100644--- a/src/metabase/notification/payload/core.clj+++ b/src/metabase/notification/payload/core.clj@@ -18,7 +18,7 @@process-virtual-dashcard][notification.payload.temp-storagecleanup!- is-cleanable?])+ cleanable?])(mr/def ::Notification"Schema for the notification."diff --git a/src/metabase/notification/payload/execute.clj b/src/metabase/notification/payload/execute.cljindex c9f22831481..7d0f3631c49 100644--- a/src/metabase/notification/payload/execute.clj+++ b/src/metabase/notification/payload/execute.clj@@ -133,21 +133,6 @@tag-names)](update-in dashcard [:visualization_settings :text] shared.params/substitute-tags tag->param (system/site-locale) (escape-markdown-chars? dashcard))))-(def ^{:private true- :doc "If a query has more than the number of rows specified here, we store the data to disk instead of in memory."}- rows-to-disk-threadhold- 1000)--(defn- data-rows-to-disk!- [qp-result context]- (if (<= (:row_count qp-result) rows-to-disk-threadhold)- (do- (log/debugf "Less than %d rows, skip storing %d rows to disk" rows-to-disk-threadhold (:row_count qp-result))- qp-result)- (do- (log/debugf "Storing %d rows to disk" (:row_count qp-result))- (update-in qp-result [:data :rows] notification.temp-storage/to-temp-file! context))))-(defn- fixup-viz-settings"The viz-settings from :data :viz-settings might be incorrect if there is a cached of the same query.See #58469.@@ -155,6 +140,10 @@[qp-result](update-in qp-result [:data :viz-settings] merge (get-in qp-result [:json_query :viz-settings])))+(def rows-to-disk-threshold+ "Maximum rows to hold in memory when running notification queries. After this, query results are streamed straight to disk. See [[metabase.notification.payload.temp-storage]] for more details."+ 2000)+(defn execute-dashboard-subscription-card"Returns subscription result for a card.@@ -187,7 +176,12 @@(^:once fn* [query info](qp(qp/userland-query query info)- nil)))))})+ ;; Pass streaming rff with 2000 row threshold+ (notification.temp-storage/notification-rff+ rows-to-disk-threshold+ {:dashboard_id dashboard_id+ :card_id card-id+ :dashcard_id (u/the-id dashcard)}))))))})result (result-fn card_id)series-results (mapv (comp result-fn :id) multi-cards)](log/debugf "Dashcard has %d series" (count multi-cards))@@ -211,11 +205,8 @@(:card_id dashcard)(log/with-context {:card_id (:card_id dashcard)}(let [parameters (merge-default-values parameters)]- ;; only do this for dashboard subscriptions but not alerts since alerts has only one card, which doesn't eat much- ;; memory- ;; TODO: we need to store series result data rows to disk too+ ;; Streaming to disk is now handled by the query processor rff(-> (execute-dashboard-subscription-card dashcard parameters)- (m/update-existing :result data-rows-to-disk! (select-keys dashcard [:dashboard_tab_id :card_id :dashboard_id]))(m/update-existing :dashcard resolve-inline-parameters parameters))))(virtual-card-of-type? dashcard "iframe")@@ -305,9 +296,12 @@(^:once fn* [query info](qp(qp/userland-query query info)- nil))))))]+ ;; Pass streaming rff with 2000 row threshold+ (notification.temp-storage/notification-rff+ rows-to-disk-threshold+ {:card-id card-id})))))))](log/debugf "Result has %d rows" (:row_count result)){:card (t2/select-one :model/Card card-id)- :result (data-rows-to-disk! result {:card-id card-id})+ :result result:type :card}))diff --git a/src/metabase/notification/payload/temp_storage.clj b/src/metabase/notification/payload/temp_storage.cljindex a8a7264193e..f7c96a6032c 100644--- a/src/metabase/notification/payload/temp_storage.clj+++ b/src/metabase/notification/payload/temp_storage.clj@@ -1,14 +1,28 @@(ns metabase.notification.payload.temp-storage- "Util to put data into a temporary file and schedule it for deletion after a specified time period.+ "Util to put data into a temporary file and delete after the notification sends. Cleanup happens with+ notification.send/do-after-notification-sent for dashboards and cards which calls cleanup! on each part. This is+ extended to Object as a no-op and on the type defined here deletes the temporary file.- Currently used to store card's rows data when sending notification since it can be large and we don't want to keep it in memory."+ When getting query results for notifications (alerts, subscriptions) once the query row count+ exceeds [[metabase.notification.payload.execute/rows-to-disk-threshold]], we then start streaming all rows to+ disk. This ensures that smaller queries don't needlessly write to disk and then reload, while large results don't+ attempt to reside in memory and kill an instance.++ The key to memory savings here is that the file will not be dereferenced if it is larger than some+ threshold. Because of this, we are safe to truncate results once the filesize goes above this+ threshold. See :notification/file-too-large and :notification/truncated?."(:require[clojure.java.io :as io]+ [metabase.analytics.prometheus :as prometheus]+ [metabase.notification.settings :as notification.settings]+ [metabase.query-processor.schema :as qp.schema]+ [metabase.util :as u][metabase.util.log :as log]+ [metabase.util.malli :as mu][metabase.util.random :as random][taoensso.nippy :as nippy])(:import- (java.io File)+ (java.io BufferedOutputStream DataOutputStream File)(java.util.concurrent Executors ScheduledThreadPoolExecutor)))(set! *warn-on-reflection* true)@@ -30,17 +44,92 @@(doto (File/createTempFile "notification-" ".npy" @temp-dir)(.deleteOnExit)))-(defn- write-to-file!- [^File file data preamble]- (nippy/freeze-to-file file {:data data :preamble preamble}))+(defn- human-readable-size [bytes]+ (cond (nil? bytes) "0 bytes"+ (zero? bytes) "0 bytes"+ :else+ (let [units ["b" "kb" "mb" "gb" "tb"]+ magnitude 1024.0]+ (loop [[unit & remaining] units+ current (double bytes)]+ (if (and (seq remaining) (> current magnitude))+ (recur remaining (/ current magnitude))+ (format "%.1f %s" current unit))))))++(defn- open-streaming-file!+ "Create and open a temp file for streaming rows. Returns map with :file and :output-stream."+ []+ (let [file (temp-file!)+ ;; make sure this is the mirror analogue of how we open it in [[read-rows-from-file]]+ os (-> (io/output-stream file)+ BufferedOutputStream.+ DataOutputStream.)]+ {:file file+ :output-stream os}))++(defn- write-row-block-to-stream!+ "Write a collection of rows to the output stream using nippy serialization."+ [^DataOutputStream os row-block]+ (nippy/freeze-to-out! os row-block)+ (.flush os))-(defn- read-from-file+(defn- read-rows-from-file+ "Read rows from a temp file. Returns vector of rows.+ Throws exception if file is larger than [[notification.settings/notification-temp-file-size-max-bytes]] unless that+ value is 0. Handles both counted and streaming formats."[^File file]- (when (.exists file)- (let [{:keys [data preamble]} (nippy/thaw-from-file file)]- (when (seq preamble)- (log/infof "reading file with preamble: %s" (pr-str preamble)))- data)))+ (when-not (.exists file)+ (throw (ex-info "Temp file no longer exists" {:file file})))++ (let [file-size (.length file)+ size-human (human-readable-size file-size)]+ (when (and (pos? (notification.settings/notification-temp-file-size-max-bytes))+ (> file-size (notification.settings/notification-temp-file-size-max-bytes)))+ (log/warnf "β οΈ SKIPPING LOAD - File too large: %s (max: %s). File will NOT be loaded into memory."+ size-human+ (human-readable-size (notification.settings/notification-temp-file-size-max-bytes)))+ (throw (ex-info "Result file too large to load into memory"+ {:type :notification/file-too-large+ :file-size file-size+ :max-size (notification.settings/notification-temp-file-size-max-bytes)+ :max-size-human-readable (human-readable-size+ (notification.settings/notification-temp-file-size-max-bytes))})))++ (log/infof "π Loading streamed results from disk: %s" size-human)++ (let [start-time (System/nanoTime)+ result (with-open [is (-> (io/input-stream file)+ java.io.BufferedInputStream.+ java.io.DataInputStream.)]+ ;; Read preamble+ (let [{:keys [preamble]} (nippy/thaw-from-in! is)]+ (when (seq preamble)+ (log/debugf "File context: %s" (pr-str preamble))))++ ;; Read row count/marker+ (let [count-or-marker (nippy/thaw-from-in! is)]+ (if (= count-or-marker ::streaming)+ ;; Streaming format - read until EOF+ (loop [rows (transient [])]+ (let [row-block (try+ (nippy/thaw-from-in! is)+ (catch java.io.EOFException _+ ::eof))]+ (if (= row-block ::eof)+ (persistent! rows)+ (recur (reduce conj! rows row-block)))))++ ;; Counted format - read exact number of rows+ (loop [rows (transient [])+ i 0]+ (if (< i count-or-marker)+ (recur (conj! rows (nippy/thaw-from-in! is))+ (inc i))+ (persistent! rows))))))+ elapsed-ms (/ (- (System/nanoTime) start-time) 1000000.0)+ row-count (count result)]+ (log/infof "β Loaded %d rows from disk in %.0f ms (%s)" row-count elapsed-ms size-human)+ result)))(.addShutdownHook(Runtime/getRuntime)@@ -56,19 +145,7 @@Object(cleanup! [_] nil))-(defn- human-readable-size [bytes]- (cond (nil? bytes) "0 bytes"- (zero? bytes) "0 bytes"- :else- (let [units ["b" "kb" "mb" "gb" "tb"]- magnitude 1024.0]- (loop [[unit & remaining] units- current (double bytes)]- (if (and (seq remaining) (> current magnitude))- (recur remaining (/ current magnitude))- (format "%.1f %s" current unit))))))--(deftype TempFileStorage [^File file]+(deftype StreamingTempFileStorage [^File file context]Cleanable(cleanup! [_](when (.exists file)@@ -76,44 +153,176 @@clojure.lang.IDeref(deref [_]- (if (.exists file)- (do- (log/infof "reading temp storage file of size: %s" (human-readable-size (.length file)))- (read-from-file file))- (throw (ex-info "File no longer exists" {:file file}))))+ (read-rows-from-file file))++ clojure.lang.IPending+ (isRealized [_]+ ;; Always return false so REPL tools like fipp won't auto-deref+ ;; Reading from disk is expensive and should be explicit+ false)Object(toString [_]- (str "#TempFileStorage{:file " file "}"))+ (if (.exists file)+ (format "#StreamingTempFileStorage{:file %s, :size %s, :context %s}"+ (.getName file)+ (human-readable-size (.length file))+ (pr-str context))+ (format "#StreamingTempFileStorage{:file %s (deleted), :context %s}"+ (.getName file)+ (pr-str context))))- ;; Add equality behavior(equals [_this other]- (and (instance? TempFileStorage other)- (= file (.file ^TempFileStorage other))))+ (and (instance? StreamingTempFileStorage other)+ (= file (.file ^StreamingTempFileStorage other))))(hashCode [_](.hashCode file)))+(defmethod print-method StreamingTempFileStorage+ [^StreamingTempFileStorage storage ^java.io.Writer w]+ (.write w (.toString storage)))++(mu/defn notification-rff :- ::qp.schema/rff+ "Reducing function factory for notifications that streams to disk when threshold exceeded.++ Returns an rff (function that takes metadata and returns an rf) that:+ - Accumulates rows in memory initially+ - Once max-row-count is reached, switches to streaming mode+ - In streaming mode, writes all accumulated rows to disk then streams subsequent rows+ - Final result contains either in-memory rows or a StreamingTempFileStorage reference++ Parameters:+ - max-row-count: Maximum number of rows to keep in memory before streaming to disk+ - context: Optional context map to include in temp file preamble (for debugging)"+ ([max-row-count]+ (notification-rff max-row-count {}))+ ([max-row-count context]+ (fn rff [metadata]+ (let [row-count (volatile! 0)+ rows (volatile! (transient []))+ streaming? (volatile! false)+ streaming-state (volatile! nil)] ; {:file File :output-stream DataOutputStream}+ (fn notification-rf+ ;; Init arity+ ([]+ {:data metadata})++ ;; Completion arity+ ([result]+ {:pre [(map? (unreduced result))]}+ (let [result (unreduced result)]+ (if @streaming?+ ;; Close the streaming file and return TempFileStorage+ (let [{:keys [^DataOutputStream output-stream ^File file]} @streaming-state]+ (try+ (when-let [remaining-rows (seq (persistent! @rows))]+ (write-row-block-to-stream! output-stream remaining-rows))+ (.close output-stream)+ (let [file-size (.length file)+ max-bytes (notification.settings/notification-temp-file-size-max-bytes)]+ (prometheus/inc! :metabase-notification/temp-storage+ {:storage (cond (:notification/truncated? @streaming-state) :truncated+ (zero? max-bytes) :not-limited+ (< max-bytes file-size) :above-threshold+ :else :disk)})+ (log/infof "πΎ Stored %d rows to disk: %s (never loaded into memory)%s"+ @row-count+ (human-readable-size file-size)+ (if (:notification/truncated? @streaming-state)+ " (note query results were truncated)"+ ""))+ (-> result+ (assoc :row_count @row-count+ :status :completed+ :data.rows-file-size file-size)+ (cond-> (:notification/truncated? @streaming-state)+ (assoc :notification/truncated? true))+ (assoc-in [:data :rows] (StreamingTempFileStorage. file context))))+ (catch Exception e+ (u/ignore-exceptions (.close output-stream))+ (throw e))))++ ;; Return in-memory rows+ (do+ (prometheus/inc! :metabase-notification/temp-storage+ {:storage :memory})+ (log/infof "β Completed with %d rows in memory (under threshold)" @row-count)+ (-> result+ (assoc :row_count @row-count+ :status :completed)+ (assoc-in [:data :rows] (persistent! @rows)))))))++ ;; Step arity - accumulate rows+ ([result row]+ ;; unconditionally increment row count and add rows to internal volatile transient collector. If we are+ ;; streaming to disk, we periodically flush this to disk; if we are collecting in memory the collection is+ ;; used the same as in the default-rff+ (vswap! row-count inc)+ (vswap! rows conj! row)+ (if @streaming?+ ;; Already streaming - write row directly to file+ (let [{:keys [^DataOutputStream output-stream ^File file]} @streaming-state]+ (if-not (zero? (mod @row-count 5000))+ ;; only flush the volatile transient every 5000 rows. If we aren't on a 5,000 row multiple, we've+ ;; already added the row above so nothing to do here.+ result+ (do+ ;; else we write the row block and then check if the size has gotten too large and we can abandon+ ;; the query with `(reduced result)`+ (write-row-block-to-stream! output-stream (persistent! @rows))+ (vreset! rows (transient []))+ (if (and (pos? (notification.settings/notification-temp-file-size-max-bytes))+ (> (.length file) (* 1.3 (notification.settings/notification-temp-file-size-max-bytes))))+ (do (vswap! streaming-state assoc :notification/truncated? true)+ (log/warnf "Results have exceeded 1.3 times of `notification-temp-file-size-max-bytes` of %s (max: %s). Truncating query results. %s"+ (human-readable-size (.length file))+ (human-readable-size (notification.settings/notification-temp-file-size-max-bytes))+ (when context (format "(%s)" (pr-str context))))+ (reduced result))+ result))))++ ;; Still in memory - check if we should start streaming+ (do+ ;; Check if we've hit the threshold to switch to disk based streaming+ (when (>= @row-count max-row-count)+ (log/infof "Row count reached threshold (%d), switching to streaming mode"+ max-row-count)+ ;; Open streaming file+ (let [{:keys [file ^DataOutputStream output-stream]} (open-streaming-file!)]+ (try+ ;; Write preamble+ (nippy/freeze-to-out! output-stream {:preamble context})+ (.flush output-stream)++ ;; Write sentinel indicating streaming format+ (nippy/freeze-to-out! output-stream ::streaming)+ (.flush output-stream)++ ;; Write all accumulated rows+ (write-row-block-to-stream! output-stream (persistent! @rows))++ ;; Switch to streaming mode+ (vreset! streaming? true)+ (vreset! rows (transient [])) ; Clear memory+ (vreset! streaming-state {:file file :output-stream output-stream})++ (catch Exception e+ (u/ignore-exceptions (.close output-stream))+ (u/ignore-exceptions (io/delete-file file))+ (throw e)))))+ result))))))))+;; ------------------------------------------------------------------------------------------------;;;; Public APIs ;;;; ------------------------------------------------------------------------------------------------;;-(defn is-cleanable?+(defn cleanable?"Returns true if x implements the Cleanable protocol"[x](satisfies? Cleanable x))-(defn to-temp-file!- "Write data to a temporary file. Returns a TempFileStorage type that:- - Implements IDeref - use @ to read the data from the file- - Implements Cleanable - call cleanup! when the file is no longer needed-- You may include an optional `preamble` which will be logged when the file is read. Helpful to leave a breadcrumb of- which dashboard/card the file might be from. "- ^TempFileStorage- ([data] (to-temp-file! data {}))- ([data preamble]- (let [f (temp-file!)]- (write-to-file! f data preamble)- (log/debug "stored data in temp file" {:length (.length ^File f)})- (TempFileStorage. f))))+(defn streaming-temp-file?+ "Check if x is a StreamingTempFileStorage instance."+ [x]+ (instance? StreamingTempFileStorage x))diff --git a/src/metabase/notification/settings.clj b/src/metabase/notification/settings.cljindex 5dc99aa44dd..b2e80479873 100644--- a/src/metabase/notification/settings.clj+++ b/src/metabase/notification/settings.clj@@ -18,3 +18,13 @@:export? false:type :integer:visibility :internal)++(defsetting notification-temp-file-size-max-bytes+ "The maximum file size that will be created when storing notification query results on disk.+ Note this is in BYTES. Default value is 10485760 which is `10 * 1024 * 1024`. To disable this size limit set the+ value to 0."+ :type :integer+ :default (* 10 1024 1024)+ :export? false+ :setter :none+ :visibility :internal)diff --git a/src/metabase/query_processor/pipeline.clj b/src/metabase/query_processor/pipeline.cljindex 7c35abee9a0..06829405d87 100644--- a/src/metabase/query_processor/pipeline.clj+++ b/src/metabase/query_processor/pipeline.clj@@ -1,6 +1,7 @@(ns metabase.query-processor.pipeline(:require[clojure.core.async :as a]+ [metabase.analytics.core :as analytics][metabase.driver :as driver][metabase.query-processor.error-type :as qp.error-type][metabase.util.i18n :as i18n]@@ -79,6 +80,8 @@(fn wrapper([] (rf))([acc]+ (analytics/inc! :metabase-query-processor/query+ {:driver driver/*driver* :status "success"})(some-> *canceled-chan* a/close!)(rf acc))([acc row]diff --git a/src/metabase/query_processor/reducible.clj b/src/metabase/query_processor/reducible.cljindex d0c9fbd08b6..12c41d926c7 100644--- a/src/metabase/query_processor/reducible.clj+++ b/src/metabase/query_processor/reducible.clj@@ -1,8 +1,6 @@(ns metabase.query-processor.reducible(:require[clojure.core.async :as a]- [metabase.analytics.core :as analytics]- [metabase.driver :as driver][metabase.lib.schema.common :as lib.schema.common][metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.schema :as qp.schema]@@ -26,7 +24,6 @@([result]{:pre [(map? (unreduced result))]};; if the result is a clojure.lang.Reduced, unwrap it so we always get back the standard-format map- (analytics/inc! :metabase-query-processor/query {:driver driver/*driver* :status "success"})(-> (unreduced result)(assoc :row_count @row-count:status :completed)diff --git a/src/metabase/query_processor/streaming.clj b/src/metabase/query_processor/streaming.cljindex 65325fa1eb5..c17a1efc257 100644--- a/src/metabase/query_processor/streaming.clj+++ b/src/metabase/query_processor/streaming.clj@@ -2,8 +2,6 @@(:refer-clojure :exclude [every? some])(:require[clojure.string :as str]- [metabase.analytics.core :as analytics]- [metabase.driver :as driver];; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use;; Lib soon^{:clj-kondo/ignore [:discouraged-namespace]}@@ -180,7 +178,6 @@{:data initial-metadata})([result]- (analytics/inc! :metabase-query-processor/query {:driver driver/*driver* :status "success"})(assoc result:row_count @row-count:status :completed))diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex 61689668b4f..448b86e8680 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -30,6 +30,7 @@[metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.metadata-providers.mock :as providers.mock]+ [metabase.notification.payload.temp-storage :as temp-storage][metabase.query-processor :as qp][metabase.query-processor.compile :as qp.compile]^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]@@ -1896,3 +1897,57 @@(doseq [[provider host] tests](let [database {:details {:host host} :engine :postgres}](is (= provider (provider-detection/detect-provider-from-database database))))))))++(deftest ^:parallel complex-types-in-notification-payload+ (mt/test-driver :postgres+ (testing "we handle complex types in notifications"+ (let [sql "SELECT+ i AS id,+ 'User_' || i AS username,+ -- JSON types+ ('{\"userId\": ' || i || ', \"score\": ' || (i % 1000) || ', \"active\": ' || (i % 2 = 0)::text || '}')::jsonb AS settings,+ -- Arrays+ ARRAY['tag' || (i % 10), 'category' || (i % 5)]::text[] AS tags,+ ARRAY[i, i*2, i*3]::integer[] AS numbers,+ -- UUID+ '7e3cd49d-bfe1-4620-83dd-0c163719175c'::uuid AS uuid,+ -- Network+ ('192.168.' || (i % 255) || '.' || ((i*7) % 255))::inet AS ip,+ -- Geometric+ point(i % 180 - 90, i % 360 - 180) AS coordinates,+ circle(point(i % 100, i % 100), 50) AS area,+ -- Text search+ to_tsvector('english', 'Content for row ' || i) AS search_data,+ -- Binary+ decode(md5(i::text), 'hex') AS hash,+ -- Range types+ int4range(i, i + 100) AS value_range,+ -- Money+ ((RANDOM() * 1000)::numeric(10,2))::money AS price,+ -- Standard+ NOW() - ((i % 365) || ' days')::interval AS created_at+ FROM generate_series(1, 5000) AS i;"+ results (qp/process-query (mt/native-query {:query sql})+ (temp-storage/notification-rff 500))]+ (is (integer? (:data.rows-file-size results)))+ (is (temp-storage/streaming-temp-file? (-> results :data :rows)))+ (is (=? [1+ "User_1"+ "{\"score\": 1, \"active\": false, \"userId\": 1}"+ ["tag1" "category1"]+ [1 2 3]+ ;; match on type+ uuid?+ "192.168.1.7"+ "(-89.0,-179.0)"+ "<(1.0,1.0),50.0>"+ "'1':4 'content':1 'row':3"+ ;; match on type+ bytes?+ "[1,101)"+ ;; match on type+ decimal?+ ;; we don't have an easy way to recognize a datetime string. assuming if it's a string and query+ ;; doesnt error then it worked+ string?]+ (-> results :data :rows deref first)))))))diff --git a/test/metabase/notification/payload/impl/card_test.clj b/test/metabase/notification/payload/impl/card_test.cljindex aba8e977adf..52ff4fc5ee1 100644--- a/test/metabase/notification/payload/impl/card_test.clj+++ b/test/metabase/notification/payload/impl/card_test.clj@@ -137,8 +137,8 @@(deftest card-with-rows-saved-to-disk-test(testing "whether the rows of a card saved to disk or in memory, all channels should work\n"(doseq [limit [1 10]]- (with-redefs [notification.payload.execute/rows-to-disk-threadhold 5]- (testing (if (> limit @#'notification.payload.execute/rows-to-disk-threadhold)+ (with-redefs [notification.payload.execute/rows-to-disk-threshold 5]+ (testing (if (> limit @#'notification.payload.execute/rows-to-disk-threshold)"card has rows saved to disk""card has rows saved in memory")(notification.tu/with-notification-testing-setup!@@ -184,7 +184,7 @@(deftest cards-with-rows-saved-to-disk-will-cleanup-the-files(let [f (atom nil)orig-execute-fn @#'notification.payload.execute/execute-card]- (with-redefs [notification.payload.execute/rows-to-disk-threadhold 1+ (with-redefs [notification.payload.execute/rows-to-disk-threshold 1notification.payload.execute/execute-card(fn [& args](let [result (apply orig-execute-fn args)]@@ -197,9 +197,9 @@:handlers [@notification.tu/default-email-handler]}](notification/send-notification! notification)(testing "sanity check that the file exists in the first place"- (is (notification.payload/is-cleanable? @f)))+ (is (notification.payload/cleanable? @f)))(testing "the files are cleaned up"- (is (not (.exists ^java.io.File (.file ^metabase.notification.payload.temp_storage.TempFileStorage @f))))))))))+ (is (not (.exists ^java.io.File (.file ^metabase.notification.payload.temp_storage.StreamingTempFileStorage @f))))))))))(deftest ensure-constraints-test(testing "Validate card queries are limited by `default-query-constraints`"diff --git a/test/metabase/notification/payload/temp_storage_test.clj b/test/metabase/notification/payload/temp_storage_test.cljindex 271d1748f7b..de7cf56ba45 100644--- a/test/metabase/notification/payload/temp_storage_test.clj+++ b/test/metabase/notification/payload/temp_storage_test.clj@@ -1,47 +1,153 @@(ns metabase.notification.payload.temp-storage-test(:require[clojure.test :refer :all]- [metabase.notification.payload.temp-storage :as temp-storage]))+ [metabase.notification.payload.temp-storage :as temp-storage]+ [metabase.test :as mt]))(set! *warn-on-reflection* true)-(deftest basic-write-read-test- (testing "basic write and read operations"- (let [test-data {:a 1 :b "test" :c [1 2 3]}- storage (temp-storage/to-temp-file! test-data)]- (try- (is (= test-data @storage))- (finally- (temp-storage/cleanup! storage))))))--(deftest cleanup-test- (testing "file is actually deleted after cleanup"- (let [storage (temp-storage/to-temp-file! "test-data")- ^java.io.File file (.file storage)]- (is (.exists file))+(defn- run-rff+ "Helper to run notification-rff with given max-rows and row data."+ [max-rows row-data]+ (let [metadata {:cols [{:name "id"} {:name "value"}]}+ rff (temp-storage/notification-rff max-rows {})+ rf (rff metadata)]+ (transduce identity rf row-data)))++(defn- rows+ "Generate n rows of test data."+ [n]+ (for [i (range n)] [i (* i 10)]))++(deftest streaming-threshold-test+ (testing "Under threshold - rows stay in memory"+ (let [result (run-rff 5 (rows 4))]+ (is (= 4 (:row_count result)))+ (is (= :completed (:status result)))+ (is (vector? (get-in result [:data :rows])))+ (is (= (rows 4) (get-in result [:data :rows])))))++ (testing "At threshold - rows stream to disk"+ (let [result (run-rff 5 (rows 5))+ storage (get-in result [:data :rows])]+ (is (= 5 (:row_count result)))+ (is (temp-storage/streaming-temp-file? storage))+ (is (= (rows 5) @storage))+ (temp-storage/cleanup! storage)))++ (testing "Over threshold - rows stream to disk"+ (let [result (run-rff 5 (rows 6))+ storage (get-in result [:data :rows])]+ (is (= 6 (:row_count result)))+ (is (temp-storage/streaming-temp-file? storage))+ (is (= (rows 6) @storage))+ (temp-storage/cleanup! storage)))++ (testing "Over max-file-size threshold, aborts query"+ (mt/with-temporary-setting-values [notification-temp-file-size-max-bytes (* 4 1024)]+ (let [many-rows 5000000+ result (run-rff 5 (rows many-rows))+ storage (get-in result [:data :rows])]+ ;; row count here is how many query results were returned+ (is (< (:row_count result) many-rows))+ (is (temp-storage/streaming-temp-file? storage))+ (is (:notification/truncated? result))+ (mt/with-temporary-setting-values [notification-temp-file-size-max-bytes 0]+ ;; remove enforcing notification size limit before dereferencing and then see that we _stored_ fewer+ ;; results than otherwise might have been returned. Without resetting this, dereferencing the rows would+ ;; error as it was larger than the file size limit++ (is (< (count @storage) many-rows))))))+ ;; in testing on my machine, this test takes this whole test from 262ms to 1550 ms. Seems worthwhile to me+ (testing "When bytes size is set to 0 allows for arbitrary sizes"+ (mt/with-temporary-setting-values [notification-temp-file-size-max-bytes 0]+ (let [many-rows 2500000+ result (run-rff 5 (rows many-rows))+ storage (get-in result [:data :rows])]+ ;; row count here is how many query results were returned+ (is (= (:row_count result) many-rows))+ (is (temp-storage/streaming-temp-file? storage))+ ;; on my machine this was 28.5mb. Don't need to push this into GB but demonstrate we are above the 10mb limit+ (is (and (:data.rows-file-size result)+ (> (:data.rows-file-size result) (* 10 1024 1024)))+ (format "data.rows-file-size was only : %d" (:data.rows-file-size result)))+ (is (not (:notification/truncated? result)))+ (is (= (count @storage) many-rows))))))++(deftest streaming-edge-cases-test+ (testing "Empty rows"+ (let [result (run-rff 5 [])]+ (is (= 0 (:row_count result)))+ (is (vector? (get-in result [:data :rows])))+ (is (empty? (get-in result [:data :rows])))))++ (testing "Single row"+ (let [result (run-rff 5 [[42]])]+ (is (= 1 (:row_count result)))+ (is (= [[42]] (get-in result [:data :rows])))))++ (testing "Large row count"+ (let [result (run-rff 100 (for [i (range 150)] [i]))+ storage (get-in result [:data :rows])]+ (is (= 150 (:row_count result)))+ (is (temp-storage/streaming-temp-file? storage))+ (is (= 150 (count @storage)))+ (is (= [0] (first @storage)))+ (is (= [149] (last @storage)))+ (temp-storage/cleanup! storage))))++(deftest streaming-data-integrity-test+ (testing "Repeated values preserve correctly"+ (let [metadata {:cols (for [c ["a" "b" "c" "d" "e"]] {:name c})}+ rff (temp-storage/notification-rff 5 {})+ rf (rff metadata)+ data (for [i (range 6)] (vec (repeat 5 i)))+ result (transduce identity rf data)+ storage (get-in result [:data :rows])]+ (is (temp-storage/streaming-temp-file? storage))+ (is (= [[0 0 0 0 0]+ [1 1 1 1 1]+ [2 2 2 2 2]+ [3 3 3 3 3]+ [4 4 4 4 4]+ [5 5 5 5 5]]+ @storage))+ (temp-storage/cleanup! storage))))++(deftest streaming-file-size-test+ (testing "File size is included in result"+ (let [result (run-rff 5 (rows 10))+ storage (get-in result [:data :rows])]+ (is (number? (:data.rows-file-size result)))+ (is (pos? (:data.rows-file-size result)))+ (is (> (:data.rows-file-size result) 50))+ (temp-storage/cleanup! storage))))++(deftest streaming-cleanup-test+ (testing "StreamingTempFileStorage can be cleaned up"+ (let [result (run-rff 5 (rows 10))+ storage (get-in result [:data :rows])]+ (is (= 10 (count @storage)))(temp-storage/cleanup! storage)- (is (not (.exists file)))- (is (thrown-with-msg?- Exception- #"File no longer exists"- @storage)))))--(deftest nil-handling-test- (testing "can handle nil values"- (let [storage (temp-storage/to-temp-file! nil)]- (try- (is (nil? @storage))- (finally- (temp-storage/cleanup! storage))))))--(deftest equality-test- (testing "equality behavior"- (let [data {:test "data"}- storage1 (temp-storage/to-temp-file! data)- storage2 (temp-storage/to-temp-file! data)]- (try- (is (= storage1 storage1))- (is (not= storage1 storage2))- (finally- (temp-storage/cleanup! storage1)- (temp-storage/cleanup! storage2))))))+ (is (thrown? Exception @storage)))))++(deftest streaming-pending-test+ (testing "IPending returns false to prevent auto-deref"+ (let [result (run-rff 5 (rows 10))+ storage (get-in result [:data :rows])]+ (is (not (realized? storage)))+ (temp-storage/cleanup! storage))))++(deftest fidelity+ (testing "Untruncated results should be equal"+ (let [query (mt/mbql-query orders)]+ ;; its the default value but just ensuring it is high enough+ (mt/with-temporary-setting-values [notification-temp-file-size-max-bytes (* 10 1024 1024)]+ (let [qp-results (mt/process-query query)+ temp-file-results (mt/process-query query+ (temp-storage/notification-rff 2000))]+ (is (not (:notification/truncated? temp-file-results)))+ (is (temp-storage/streaming-temp-file? (-> temp-file-results :data :rows)))+ (is (= (-> qp-results :data :rows)+ (-> temp-file-results :data :rows deref)))+ (-> temp-file-results :data :rows temp-storage/cleanup!))))))diff --git a/test/metabase/pulse/dashboard_subscription_test.clj b/test/metabase/pulse/dashboard_subscription_test.cljindex 27a1c91b20b..78f58368e30 100644--- a/test/metabase/pulse/dashboard_subscription_test.clj+++ b/test/metabase/pulse/dashboard_subscription_test.clj@@ -1287,7 +1287,7 @@(str "metabase-test" (random/random-name)))](.mkdirs dir)dir))- notification.payload.execute/rows-to-disk-threadhold 1+ notification.payload.execute/rows-to-disk-threshold 1channel/send! (fn [& _args](testing "sanity check that there are files there to cleanup"(is (not-empty (.listFiles ^java.io.File @@#'notification.temp-storage/temp-dir)))))]@@ -1302,9 +1302,9 @@(deftest dashboard-with-rows-saved-to-disk-test(testing "whether the rows of a dashboard saved to disk or in memory, all channels should work"(doseq [limit [1 #_10]]- (with-redefs [notification.payload.execute/rows-to-disk-threadhold 5]+ (with-redefs [notification.payload.execute/rows-to-disk-threshold 5](notification.tu/with-channel-fixtures [:channel/slack]- (testing (if (> limit @#'notification.payload.execute/rows-to-disk-threadhold)+ (testing (if (> limit @#'notification.payload.execute/rows-to-disk-threshold)"dashboard has rows saved to disk""dashboard has rows saved in memory")(mt/with-temp [:model/Card {card-id :id} {:name pulse.test-util/card-namediff --git a/test/metabase/pulse/render/test_util.clj b/test/metabase/pulse/render/test_util.cljindex 7fa12b51f80..b8ef638735d 100644--- a/test/metabase/pulse/render/test_util.clj+++ b/test/metabase/pulse/render/test_util.clj@@ -12,6 +12,7 @@[metabase.channel.render.core :as channel.render][metabase.channel.render.image-bundle :as image-bundle][metabase.channel.render.js.svg :as js.svg]+ [metabase.channel.shared :as channel.shared][metabase.notification.payload.execute :as notification.execute][metabase.query-processor :as qp][metabase.query-processor.card :as qp.card]@@ -182,7 +183,8 @@([dashcard-id parameters](let [dashcard (t2/select-one :model/DashboardCard :id dashcard-id)card (t2/select-one :model/Card :id (:card_id dashcard))- {:keys [result dashcard]} (notification.execute/execute-dashboard-subscription-card dashcard parameters)]+ {:keys [result dashcard]} (channel.shared/maybe-realize-data-rows+ (notification.execute/execute-dashboard-subscription-card dashcard parameters))](with-redefs [js.svg/svg-string->bytes identityimage-bundle/make-image-bundle (fn [_ s]{:image-src s