The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Cell count for notifications temp storage (#65518)
dpsutton00ee60d8ed23
diff --git a/src/metabase/notification/payload/execute.clj b/src/metabase/notification/payload/execute.cljindex be72cb84104..58e47f6d9c6 100644--- a/src/metabase/notification/payload/execute.clj+++ b/src/metabase/notification/payload/execute.clj@@ -142,9 +142,10 @@(update-in qp-result [:data :viz-settings] merge (-> (get-in qp-result [:json_query :viz-settings])viz-settings/db->norm)))-(def rows-to-disk-threshold- "Maximum rows to hold in memory when running notification queries. After this, query results are streamed straight to disk. See [[metabase.notification.payload.temp-storage]] for more details."- 2000)+(def cells-to-disk-threshold+ "Maximum cells (rows * columns) to hold in memory when running notification queries. After this, query results are+ streamed straight to disk. See [[metabase.notification.payload.temp-storage]] for more details."+ 20000)(defn execute-dashboard-subscription-card"Returns subscription result for a card.@@ -180,7 +181,7 @@(qp/userland-query query info);; Pass streaming rff with 2000 row threshold(notification.temp-storage/notification-rff- rows-to-disk-threshold+ cells-to-disk-threshold{:dashboard_id dashboard_id:card_id card-id:dashcard_id (u/the-id dashcard)}))))))})@@ -300,7 +301,7 @@(qp/userland-query query info);; Pass streaming rff with 2000 row threshold(notification.temp-storage/notification-rff- rows-to-disk-threshold+ cells-to-disk-threshold{:card-id card-id})))))))](log/debugf "Result has %d rows" (:row_count result))diff --git a/src/metabase/notification/payload/temp_storage.clj b/src/metabase/notification/payload/temp_storage.cljindex f7c96a6032c..68cf4418119 100644--- a/src/metabase/notification/payload/temp_storage.clj+++ b/src/metabase/notification/payload/temp_storage.clj@@ -4,7 +4,7 @@extended to Object as a no-op and on the type defined here deletes the temporary file.When getting query results for notifications (alerts, subscriptions) once the query row count- exceeds [[metabase.notification.payload.execute/rows-to-disk-threshold]], we then start streaming all rows to+ exceeds [[metabase.notification.payload.execute/cells-to-disk-threshold]], we then start streaming all rows todisk. This ensures that smaller queries don't needlessly write to disk and then reload, while large results don'tattempt to reside in memory and kill an instance.@@ -62,16 +62,15 @@(let [file (temp-file!);; make sure this is the mirror analogue of how we open it in [[read-rows-from-file]]os (-> (io/output-stream file)- BufferedOutputStream.+ (BufferedOutputStream. (* 8 1024))DataOutputStream.)]{:file file:output-stream os}))-(defn- write-row-block-to-stream!+(defn- write-row-to-stream!"Write a collection of rows to the output stream using nippy serialization."- [^DataOutputStream os row-block]- (nippy/freeze-to-out! os row-block)- (.flush os))+ [^DataOutputStream os row]+ (nippy/freeze-to-out! os row))(defn- read-rows-from-file"Read rows from a temp file. Returns vector of rows.@@ -111,13 +110,13 @@(if (= count-or-marker ::streaming);; Streaming format - read until EOF(loop [rows (transient [])]- (let [row-block (try- (nippy/thaw-from-in! is)- (catch java.io.EOFException _- ::eof))]- (if (= row-block ::eof)+ (let [row-read (try+ (nippy/thaw-from-in! is)+ (catch java.io.EOFException _+ ::eof))]+ (if (= row-read ::eof)(persistent! rows)- (recur (reduce conj! rows row-block)))))+ (recur (conj! rows row-read)))));; Counted format - read exact number of rows(loop [rows (transient [])@@ -188,18 +187,20 @@Returns an rff (function that takes metadata and returns an rf) that:- Accumulates rows in memory initially- - Once max-row-count is reached, switches to streaming mode+ - Once max-cell-count is reached, switches to streaming mode- In streaming mode, writes all accumulated rows to disk then streams subsequent rows- Final result contains either in-memory rows or a StreamingTempFileStorage referenceParameters:- - max-row-count: Maximum number of rows to keep in memory before streaming to disk++ - max-cell-count: Maximum number of \"cells\" or values to keep in memory before streaming to disk.- context: Optional context map to include in temp file preamble (for debugging)"- ([max-row-count]- (notification-rff max-row-count {}))- ([max-row-count context]+ ([max-cell-count]+ (notification-rff max-cell-count {}))+ ([max-cell-count context](fn rff [metadata](let [row-count (volatile! 0)+ cell-count (volatile! 0)rows (volatile! (transient []))streaming? (volatile! false)streaming-state (volatile! nil)] ; {:file File :output-stream DataOutputStream}@@ -216,8 +217,6 @@;; Close the streaming file and return TempFileStorage(let [{:keys [^DataOutputStream output-stream ^File file]} @streaming-state](try- (when-let [remaining-rows (seq (persistent! @rows))]- (write-row-block-to-stream! output-stream remaining-rows))(.close output-stream)(let [file-size (.length file)max-bytes (notification.settings/notification-temp-file-size-max-bytes)]@@ -259,35 +258,27 @@;; streaming to disk, we periodically flush this to disk; if we are collecting in memory the collection is;; used the same as in the default-rff(vswap! row-count inc)- (vswap! rows conj! row)(if @streaming?;; Already streaming - write row directly to file(let [{:keys [^DataOutputStream output-stream ^File file]} @streaming-state]- (if-not (zero? (mod @row-count 5000))- ;; only flush the volatile transient every 5000 rows. If we aren't on a 5,000 row multiple, we've- ;; already added the row above so nothing to do here.- result- (do- ;; else we write the row block and then check if the size has gotten too large and we can abandon- ;; the query with `(reduced result)`- (write-row-block-to-stream! output-stream (persistent! @rows))- (vreset! rows (transient []))- (if (and (pos? (notification.settings/notification-temp-file-size-max-bytes))- (> (.length file) (* 1.3 (notification.settings/notification-temp-file-size-max-bytes))))- (do (vswap! streaming-state assoc :notification/truncated? true)- (log/warnf "Results have exceeded 1.3 times of `notification-temp-file-size-max-bytes` of %s (max: %s). Truncating query results. %s"- (human-readable-size (.length file))- (human-readable-size (notification.settings/notification-temp-file-size-max-bytes))- (when context (format "(%s)" (pr-str context))))- (reduced result))- result))))-- ;; Still in memory - check if we should start streaming+ (write-row-to-stream! output-stream row)+ (if (and (pos? (notification.settings/notification-temp-file-size-max-bytes))+ (> (.length file) (* 1.3 (notification.settings/notification-temp-file-size-max-bytes))))+ (do (vswap! streaming-state assoc :notification/truncated? true)+ (log/warnf "Results have exceeded 1.3 times of `notification-temp-file-size-max-bytes` of %s (max: %s). Truncating query results. %s"+ (human-readable-size (.length file))+ (human-readable-size (notification.settings/notification-temp-file-size-max-bytes))+ (when context (format "(%s)" (pr-str context))))+ (reduced result))+ result))+(do+ (vswap! rows conj! row)+ (vswap! cell-count #(+ % (count row)));; Check if we've hit the threshold to switch to disk based streaming- (when (>= @row-count max-row-count)- (log/infof "Row count reached threshold (%d), switching to streaming mode"- max-row-count)+ (when (>= @cell-count max-cell-count)+ (log/infof "Cell count reached threshold (%d, %d rows), switching to streaming mode"+ max-cell-count @row-count);; Open streaming file(let [{:keys [file ^DataOutputStream output-stream]} (open-streaming-file!)](try@@ -300,7 +291,8 @@(.flush output-stream);; Write all accumulated rows- (write-row-block-to-stream! output-stream (persistent! @rows))+ (doseq [row (persistent! @rows)]+ (write-row-to-stream! output-stream row));; Switch to streaming mode(vreset! streaming? true)diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex 9323843e0b6..f2a83e2e2ff 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -1913,7 +1913,8 @@NOW() - ((i % 365) || ' days')::interval AS created_atFROM generate_series(1, 5000) AS i;"results (qp/process-query (mt/native-query {:query sql})- (temp-storage/notification-rff 500))]+ (temp-storage/notification-rff+ 5000 {:context 'complex-types-in-notification-payload}))](is (integer? (:data.rows-file-size results)))(is (temp-storage/streaming-temp-file? (-> results :data :rows)))(is (=? [1diff --git a/test/metabase/notification/payload/impl/card_test.clj b/test/metabase/notification/payload/impl/card_test.cljindex 52ff4fc5ee1..1629f45a99c 100644--- a/test/metabase/notification/payload/impl/card_test.clj+++ b/test/metabase/notification/payload/impl/card_test.clj@@ -137,8 +137,8 @@(deftest card-with-rows-saved-to-disk-test(testing "whether the rows of a card saved to disk or in memory, all channels should work\n"(doseq [limit [1 10]]- (with-redefs [notification.payload.execute/rows-to-disk-threshold 5]- (testing (if (> limit @#'notification.payload.execute/rows-to-disk-threshold)+ (with-redefs [notification.payload.execute/cells-to-disk-threshold 5]+ (testing (if (> limit @#'notification.payload.execute/cells-to-disk-threshold)"card has rows saved to disk""card has rows saved in memory")(notification.tu/with-notification-testing-setup!@@ -184,7 +184,7 @@(deftest cards-with-rows-saved-to-disk-will-cleanup-the-files(let [f (atom nil)orig-execute-fn @#'notification.payload.execute/execute-card]- (with-redefs [notification.payload.execute/rows-to-disk-threshold 1+ (with-redefs [notification.payload.execute/cells-to-disk-threshold 1notification.payload.execute/execute-card(fn [& args](let [result (apply orig-execute-fn args)]diff --git a/test/metabase/notification/payload/temp_storage_test.clj b/test/metabase/notification/payload/temp_storage_test.cljindex de7cf56ba45..da53f4375e5 100644--- a/test/metabase/notification/payload/temp_storage_test.clj+++ b/test/metabase/notification/payload/temp_storage_test.clj@@ -17,18 +17,18 @@(defn- rows"Generate n rows of test data."[n]- (for [i (range n)] [i (* i 10)]))+ (for [i (range n)] (repeat 10 (* 10 i))))(deftest streaming-threshold-test(testing "Under threshold - rows stay in memory"- (let [result (run-rff 5 (rows 4))]+ (let [result (run-rff 50 (rows 4))](is (= 4 (:row_count result)))(is (= :completed (:status result)))(is (vector? (get-in result [:data :rows])))(is (= (rows 4) (get-in result [:data :rows])))))(testing "At threshold - rows stream to disk"- (let [result (run-rff 5 (rows 5))+ (let [result (run-rff 50 (rows 5))storage (get-in result [:data :rows])](is (= 5 (:row_count result)))(is (temp-storage/streaming-temp-file? storage))@@ -36,7 +36,7 @@(temp-storage/cleanup! storage)))(testing "Over threshold - rows stream to disk"- (let [result (run-rff 5 (rows 6))+ (let [result (run-rff 50 (rows 6))storage (get-in result [:data :rows])](is (= 6 (:row_count result)))(is (temp-storage/streaming-temp-file? storage))@@ -45,7 +45,7 @@(testing "Over max-file-size threshold, aborts query"(mt/with-temporary-setting-values [notification-temp-file-size-max-bytes (* 4 1024)]- (let [many-rows 5000000+ (let [many-rows 50000result (run-rff 5 (rows many-rows))storage (get-in result [:data :rows])];; row count here is how many query results were returned@@ -61,8 +61,8 @@;; in testing on my machine, this test takes this whole test from 262ms to 1550 ms. Seems worthwhile to me(testing "When bytes size is set to 0 allows for arbitrary sizes"(mt/with-temporary-setting-values [notification-temp-file-size-max-bytes 0]- (let [many-rows 2500000- result (run-rff 5 (rows many-rows))+ (let [many-rows 250000+ result (run-rff 50 (rows many-rows))storage (get-in result [:data :rows])];; row count here is how many query results were returned(is (= (:row_count result) many-rows))@@ -145,7 +145,7 @@(mt/with-temporary-setting-values [notification-temp-file-size-max-bytes (* 10 1024 1024)](let [qp-results (mt/process-query query)temp-file-results (mt/process-query query- (temp-storage/notification-rff 2000))]+ (temp-storage/notification-rff 20000))](is (not (:notification/truncated? temp-file-results)))(is (temp-storage/streaming-temp-file? (-> temp-file-results :data :rows)))(is (= (-> qp-results :data :rows)diff --git a/test/metabase/pulse/dashboard_subscription_test.clj b/test/metabase/pulse/dashboard_subscription_test.cljindex 78f58368e30..a9f6b775259 100644--- a/test/metabase/pulse/dashboard_subscription_test.clj+++ b/test/metabase/pulse/dashboard_subscription_test.clj@@ -1287,7 +1287,7 @@(str "metabase-test" (random/random-name)))](.mkdirs dir)dir))- notification.payload.execute/rows-to-disk-threshold 1+ notification.payload.execute/cells-to-disk-threshold 1channel/send! (fn [& _args](testing "sanity check that there are files there to cleanup"(is (not-empty (.listFiles ^java.io.File @@#'notification.temp-storage/temp-dir)))))]@@ -1302,9 +1302,9 @@(deftest dashboard-with-rows-saved-to-disk-test(testing "whether the rows of a dashboard saved to disk or in memory, all channels should work"(doseq [limit [1 #_10]]- (with-redefs [notification.payload.execute/rows-to-disk-threshold 5]+ (with-redefs [notification.payload.execute/cells-to-disk-threshold 5](notification.tu/with-channel-fixtures [:channel/slack]- (testing (if (> limit @#'notification.payload.execute/rows-to-disk-threshold)+ (testing (if (> limit @#'notification.payload.execute/cells-to-disk-threshold)"dashboard has rows saved to disk""dashboard has rows saved in memory")(mt/with-temp [:model/Card {card-id :id} {:name pulse.test-util/card-name