Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Simplify notification result data flow (#70115)

Ngoc Khuataa66ab25d6db
diff --git a/src/metabase/notification/payload/execute.clj b/src/metabase/notification/payload/execute.clj
index 643947ff80a..78a8185f9cc 100644
--- a/src/metabase/notification/payload/execute.clj
+++ b/src/metabase/notification/payload/execute.clj
@@ -142,6 +142,17 @@
(update-in qp-result [:data :viz-settings] merge (-> (get-in qp-result [:json_query :viz-settings])
viz-settings/db->norm)))
+(defn- format-qp-result
+ "Extracts only the keys that the notification pipeline needs from a QP result.
+ The full QP result contains many internal bookkeeping keys that are irrelevant
+ to rendering notifications, so we select just the ones downstream consumers use."
+ [qp-result]
+ (-> (select-keys qp-result [:status :row_count :database_id :error
+ :notification/truncated? :data.rows-file-size])
+ (assoc :data (select-keys (:data qp-result)
+ [:cols :rows :viz-settings :results_metadata :insights
+ :results_timezone :format-rows? :pivot-export-options]))))
+
(def cells-to-disk-threshold
"Maximum cells (rows * columns) to hold in memory when running notification queries. After this, query results are
streamed straight to disk. See [[metabase.notification.payload.temp-storage]] for more details."
@@ -163,28 +174,29 @@
:dashcard dashcard
;; TODO should this be dashcard?
:type :card
- :result (fixup-viz-settings
- (qp.dashboard/process-query-for-dashcard
- :dashboard-id dashboard_id
- :card-id card-id
- :dashcard-id (u/the-id dashcard)
- :context :dashboard-subscription
- :export-format :api
- :parameters parameters
- :constraints {}
- :middleware {:process-viz-settings? true
- :js-int-to-string? false
- :add-default-userland-constraints? false}
- :make-run (fn make-run [qp _export-format]
- (^:once fn* [query info]
- (qp
- (qp/userland-query query info)
+ :result (-> (qp.dashboard/process-query-for-dashcard
+ :dashboard-id dashboard_id
+ :card-id card-id
+ :dashcard-id (u/the-id dashcard)
+ :context :dashboard-subscription
+ :export-format :api
+ :parameters parameters
+ :constraints {}
+ :middleware {:process-viz-settings? true
+ :js-int-to-string? false
+ :add-default-userland-constraints? false}
+ :make-run (fn make-run [qp _export-format]
+ (^:once fn* [query info]
+ (qp
+ (qp/userland-query query info)
;; Pass streaming rff with 2000 row threshold
- (notification.temp-storage/notification-rff
- cells-to-disk-threshold
- {:dashboard_id dashboard_id
- :card_id card-id
- :dashcard_id (u/the-id dashcard)}))))))})
+ (notification.temp-storage/notification-rff
+ cells-to-disk-threshold
+ {:dashboard_id dashboard_id
+ :card_id card-id
+ :dashcard_id (u/the-id dashcard)})))))
+ fixup-viz-settings
+ format-qp-result)})
result (result-fn card_id)
series-results (mapv (comp result-fn :id) multi-cards)]
(log/debugf "Dashcard has %d series" (count multi-cards))
@@ -286,23 +298,24 @@
[creator-id :- pos-int?
card-id :- pos-int?]
(let [result (request/with-current-user creator-id
- (fixup-viz-settings
- (qp.card/process-query-for-card card-id :api
- ;; TODO rename to :notification?
- :context :pulse
- :constraints {}
- :middleware {:skip-results-metadata? false
- :process-viz-settings? true
- :js-int-to-string? false
- :add-default-userland-constraints? false}
- :make-run (fn make-run [qp _export-format]
- (^:once fn* [query info]
- (qp
- (qp/userland-query query info)
- ;; Pass streaming rff with 2000 row threshold
- (notification.temp-storage/notification-rff
- cells-to-disk-threshold
- {:card-id card-id})))))))]
+ (-> (qp.card/process-query-for-card card-id :api
+ ;; TODO rename to :notification?
+ :context :pulse
+ :constraints {}
+ :middleware {:skip-results-metadata? false
+ :process-viz-settings? true
+ :js-int-to-string? false
+ :add-default-userland-constraints? false}
+ :make-run (fn make-run [qp _export-format]
+ (^:once fn* [query info]
+ (qp
+ (qp/userland-query query info)
+ ;; Pass streaming rff with 2000 row threshold
+ (notification.temp-storage/notification-rff
+ cells-to-disk-threshold
+ {:card-id card-id})))))
+ fixup-viz-settings
+ format-qp-result))]
(log/debugf "Result has %d rows" (:row_count result))
{:card (t2/select-one :model/Card card-id)
diff --git a/test/metabase/notification/payload/core_test.clj b/test/metabase/notification/payload/core_test.clj
new file mode 100644
index 00000000000..3d9caae8e31
--- /dev/null
+++ b/test/metabase/notification/payload/core_test.clj
@@ -0,0 +1,21 @@
+(ns metabase.notification.payload.core-test
+ (:require
+ [clojure.test :refer :all]
+ [clojure.walk :as walk]
+ [metabase.notification.core :as notification]
+ [metabase.notification.test-util :as notification.tu]))
+
+(set! *warn-on-reflection* true)
+
+(deftest lib-metadata-stripped-from-payload-test
+ (testing "lib/metadata keys are stripped from the realized notification payload"
+ (notification.tu/with-card-notification
+ [notification {}]
+ (let [payload (notification/notification-payload notification)]
+ (walk/prewalk
+ (fn [x]
+ (when (map? x)
+ (is (not (contains? x :lib/metadata))
+ "No :lib/metadata should survive in the notification payload"))
+ x)
+ payload)))))
diff --git a/test/metabase/notification/payload/execute_test.clj b/test/metabase/notification/payload/execute_test.clj
new file mode 100644
index 00000000000..758e3951cf1
--- /dev/null
+++ b/test/metabase/notification/payload/execute_test.clj
@@ -0,0 +1,50 @@
+(ns metabase.notification.payload.execute-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.notification.payload.execute :as notification.payload.execute]
+ [metabase.test :as mt]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private expected-top-level-keys
+ "The only top-level keys that should appear in a notification result."
+ #{:status :row_count :database_id :error :data :notification/truncated? :data.rows-file-size})
+
+(def ^:private expected-data-keys
+ "The only keys that should appear in the :data sub-map of a notification result."
+ #{:cols :rows :viz-settings :results_metadata :insights :results_timezone
+ :format-rows? :pivot-export-options})
+
+(defn- check-result-structure
+ [result]
+ (is (every? expected-top-level-keys (keys result))
+ "Result should only contain expected top-level keys")
+ (is (not (contains? result :json_query))
+ ":json_query must not leak into notification results")
+ (is (every? expected-data-keys (keys (:data result)))
+ ":data should only contain expected keys")
+ (is (not (contains? (:data result) :native_form))
+ ":native_form must not leak into notification results")
+ (is (= :completed (:status result)))
+ (is (pos? (:row_count result))))
+
+(deftest execute-card-result-structure-test
+ (testing "execute-card returns only the expected keys"
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query venues)}]
+ (let [part (notification.payload.execute/execute-card (mt/user->id :rasta) card-id)
+ result (:result part)]
+ (is (= :card (:type part)))
+ (check-result-structure result)))))
+
+(deftest execute-dashboard-subscription-card-result-structure-test
+ (testing "execute-dashboard-subscription-card returns only the expected keys"
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query venues)}
+ :model/Dashboard {dash-id :id} {}
+ :model/DashboardCard {_dc-id :id
+ :as dashcard} {:dashboard_id dash-id
+ :card_id card-id}]
+ (mt/with-current-user (mt/user->id :rasta)
+ (let [part (notification.payload.execute/execute-dashboard-subscription-card dashcard [])
+ result (:result part)]
+ (is (= :card (:type part)))
+ (check-result-structure result))))))