The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Simplify notification result data flow (#70115)
Ngoc Khuataa66ab25d6db
diff --git a/src/metabase/notification/payload/execute.clj b/src/metabase/notification/payload/execute.cljindex 643947ff80a..78a8185f9cc 100644--- a/src/metabase/notification/payload/execute.clj+++ b/src/metabase/notification/payload/execute.clj@@ -142,6 +142,17 @@(update-in qp-result [:data :viz-settings] merge (-> (get-in qp-result [:json_query :viz-settings])viz-settings/db->norm)))+(defn- format-qp-result+ "Extracts only the keys that the notification pipeline needs from a QP result.+ The full QP result contains many internal bookkeeping keys that are irrelevant+ to rendering notifications, so we select just the ones downstream consumers use."+ [qp-result]+ (-> (select-keys qp-result [:status :row_count :database_id :error+ :notification/truncated? :data.rows-file-size])+ (assoc :data (select-keys (:data qp-result)+ [:cols :rows :viz-settings :results_metadata :insights+ :results_timezone :format-rows? :pivot-export-options]))))+(def cells-to-disk-threshold"Maximum cells (rows * columns) to hold in memory when running notification queries. After this, query results arestreamed straight to disk. See [[metabase.notification.payload.temp-storage]] for more details."@@ -163,28 +174,29 @@:dashcard dashcard;; TODO should this be dashcard?:type :card- :result (fixup-viz-settings- (qp.dashboard/process-query-for-dashcard- :dashboard-id dashboard_id- :card-id card-id- :dashcard-id (u/the-id dashcard)- :context :dashboard-subscription- :export-format :api- :parameters parameters- :constraints {}- :middleware {:process-viz-settings? true- :js-int-to-string? false- :add-default-userland-constraints? false}- :make-run (fn make-run [qp _export-format]- (^:once fn* [query info]- (qp- (qp/userland-query query info)+ :result (-> (qp.dashboard/process-query-for-dashcard+ :dashboard-id dashboard_id+ :card-id card-id+ :dashcard-id (u/the-id dashcard)+ :context :dashboard-subscription+ :export-format :api+ :parameters parameters+ :constraints {}+ :middleware {:process-viz-settings? true+ :js-int-to-string? false+ :add-default-userland-constraints? false}+ :make-run (fn make-run [qp _export-format]+ (^:once fn* [query info]+ (qp+ (qp/userland-query query info);; Pass streaming rff with 2000 row threshold- (notification.temp-storage/notification-rff- cells-to-disk-threshold- {:dashboard_id dashboard_id- :card_id card-id- :dashcard_id (u/the-id dashcard)}))))))})+ (notification.temp-storage/notification-rff+ cells-to-disk-threshold+ {:dashboard_id dashboard_id+ :card_id card-id+ :dashcard_id (u/the-id dashcard)})))))+ fixup-viz-settings+ format-qp-result)})result (result-fn card_id)series-results (mapv (comp result-fn :id) multi-cards)](log/debugf "Dashcard has %d series" (count multi-cards))@@ -286,23 +298,24 @@[creator-id :- pos-int?card-id :- pos-int?](let [result (request/with-current-user creator-id- (fixup-viz-settings- (qp.card/process-query-for-card card-id :api- ;; TODO rename to :notification?- :context :pulse- :constraints {}- :middleware {:skip-results-metadata? false- :process-viz-settings? true- :js-int-to-string? false- :add-default-userland-constraints? false}- :make-run (fn make-run [qp _export-format]- (^:once fn* [query info]- (qp- (qp/userland-query query info)- ;; Pass streaming rff with 2000 row threshold- (notification.temp-storage/notification-rff- cells-to-disk-threshold- {:card-id card-id})))))))]+ (-> (qp.card/process-query-for-card card-id :api+ ;; TODO rename to :notification?+ :context :pulse+ :constraints {}+ :middleware {:skip-results-metadata? false+ :process-viz-settings? true+ :js-int-to-string? false+ :add-default-userland-constraints? false}+ :make-run (fn make-run [qp _export-format]+ (^:once fn* [query info]+ (qp+ (qp/userland-query query info)+ ;; Pass streaming rff with 2000 row threshold+ (notification.temp-storage/notification-rff+ cells-to-disk-threshold+ {:card-id card-id})))))+ fixup-viz-settings+ format-qp-result))](log/debugf "Result has %d rows" (:row_count result)){:card (t2/select-one :model/Card card-id)diff --git a/test/metabase/notification/payload/core_test.clj b/test/metabase/notification/payload/core_test.cljnew file mode 100644index 00000000000..3d9caae8e31--- /dev/null+++ b/test/metabase/notification/payload/core_test.clj@@ -0,0 +1,21 @@+(ns metabase.notification.payload.core-test+ (:require+ [clojure.test :refer :all]+ [clojure.walk :as walk]+ [metabase.notification.core :as notification]+ [metabase.notification.test-util :as notification.tu]))++(set! *warn-on-reflection* true)++(deftest lib-metadata-stripped-from-payload-test+ (testing "lib/metadata keys are stripped from the realized notification payload"+ (notification.tu/with-card-notification+ [notification {}]+ (let [payload (notification/notification-payload notification)]+ (walk/prewalk+ (fn [x]+ (when (map? x)+ (is (not (contains? x :lib/metadata))+ "No :lib/metadata should survive in the notification payload"))+ x)+ payload)))))diff --git a/test/metabase/notification/payload/execute_test.clj b/test/metabase/notification/payload/execute_test.cljnew file mode 100644index 00000000000..758e3951cf1--- /dev/null+++ b/test/metabase/notification/payload/execute_test.clj@@ -0,0 +1,50 @@+(ns metabase.notification.payload.execute-test+ (:require+ [clojure.test :refer :all]+ [metabase.notification.payload.execute :as notification.payload.execute]+ [metabase.test :as mt]))++(set! *warn-on-reflection* true)++(def ^:private expected-top-level-keys+ "The only top-level keys that should appear in a notification result."+ #{:status :row_count :database_id :error :data :notification/truncated? :data.rows-file-size})++(def ^:private expected-data-keys+ "The only keys that should appear in the :data sub-map of a notification result."+ #{:cols :rows :viz-settings :results_metadata :insights :results_timezone+ :format-rows? :pivot-export-options})++(defn- check-result-structure+ [result]+ (is (every? expected-top-level-keys (keys result))+ "Result should only contain expected top-level keys")+ (is (not (contains? result :json_query))+ ":json_query must not leak into notification results")+ (is (every? expected-data-keys (keys (:data result)))+ ":data should only contain expected keys")+ (is (not (contains? (:data result) :native_form))+ ":native_form must not leak into notification results")+ (is (= :completed (:status result)))+ (is (pos? (:row_count result))))++(deftest execute-card-result-structure-test+ (testing "execute-card returns only the expected keys"+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query venues)}]+ (let [part (notification.payload.execute/execute-card (mt/user->id :rasta) card-id)+ result (:result part)]+ (is (= :card (:type part)))+ (check-result-structure result)))))++(deftest execute-dashboard-subscription-card-result-structure-test+ (testing "execute-dashboard-subscription-card returns only the expected keys"+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query venues)}+ :model/Dashboard {dash-id :id} {}+ :model/DashboardCard {_dc-id :id+ :as dashcard} {:dashboard_id dash-id+ :card_id card-id}]+ (mt/with-current-user (mt/user->id :rasta)+ (let [part (notification.payload.execute/execute-dashboard-subscription-card dashcard [])+ result (:result part)]+ (is (= :card (:type part)))+ (check-result-structure result))))))