The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Skip null values when analyzing a chart (#84261)
Andrei Simionescua1ca09fcfd9e
diff --git a/src/metabase/interestingness/chart.clj b/src/metabase/interestingness/chart.cljindex ad2d40b4855..be9a7cf86fb 100644--- a/src/metabase/interestingness/chart.clj+++ b/src/metabase/interestingness/chart.clj@@ -168,8 +168,9 @@(chart-interestingness chart-config(chart.stats/compute-chart-stats chart-config {})))([chart-config stats]- (let [nd (non-degeneracy-score chart-config)- chart-type (:chart-type stats)]+ (let [chart-config (assoc chart-config :series (chart.stats/analyzable-series chart-config))+ nd (non-degeneracy-score chart-config)+ chart-type (:chart-type stats)](if (zero? nd){:score 0.0 :non-degeneracy nd :signal nil :structure nil :chart-type chart-type}(let [sig (signal-score stats)diff --git a/src/metabase/interestingness/chart/stats.clj b/src/metabase/interestingness/chart/stats.cljindex b19f3db8b83..0dc8c19d16d 100644--- a/src/metabase/interestingness/chart/stats.clj+++ b/src/metabase/interestingness/chart/stats.clj@@ -1,6 +1,7 @@(ns metabase.interestingness.chart.stats"Chart type detection and statistics routing."(:require+ [medley.core :as m][metabase.interestingness.chart.categorical :as categorical][metabase.interestingness.chart.histogram :as histogram][metabase.interestingness.chart.scatter :as scatter]@@ -56,6 +57,20 @@(when (seq downsampled){:downsampled-series downsampled})]))+(defn- drop-nil-points+ "Drop the points of `series-config` whose x or y value is nil."+ [{x-values :x_values, y-values :y_values, :as series-config}]+ (let [points (filter (fn [[x y]] (and (some? x) (some? y))) (map vector x-values y-values))]+ (assoc series-config+ :x_values (mapv first points)+ :y_values (mapv second points))))++(defn analyzable-series+ "Return the series of `chart-config` with their nil points dropped, leaving out any series with no points left."+ [chart-config]+ (->> (update-vals (:series chart-config) drop-nil-points)+ (m/filter-vals (comp seq :y_values))))+;;; ----------------------------------------------- Chart Type Detection ---------------------------------------------(def ^:private explicit-display-types@@ -138,7 +153,7 @@(mu/defn compute-chart-stats :- ::stats.types/chart-stats"Compute statistics for a chart based on its detected type.- Applies data limits before computation:+ Drops points with a nil x or y value and series left empty, then applies data limits before computation:- Each series is downsampled to at most [[max-data-points-per-series]] points- Correlation computation is limited to [[max-series-for-correlations]] series@@ -148,8 +163,9 @@:deep? - compute additional statistics"[chart-config :- ::stats.types/chart-configopts :- ::stats.types/options]- (let [chart-type (detect-chart-type chart-config)- [limited-series limits-info] (apply-data-limits (:series chart-config))+ (let [series (analyzable-series chart-config)+ chart-type (detect-chart-type (assoc chart-config :series series))+ [limited-series limits-info] (apply-data-limits series)too-many-series? (> (count limited-series) max-series-for-correlations)opts (cond-> optstoo-many-series? (assoc :max-correlation-series max-series-for-correlations))diff --git a/src/metabase/interestingness/chart/types.clj b/src/metabase/interestingness/chart/types.cljindex 99a7c45f8c1..d82e46e3174 100644--- a/src/metabase/interestingness/chart/types.clj+++ b/src/metabase/interestingness/chart/types.clj@@ -32,6 +32,14 @@[:chart_type {:optional true} [:maybe :string]][:stacked {:optional true} [:maybe :boolean]]])+(mr/def ::raw-series-config+ "A series as [[metabase.interestingness.chart.stats/compute-chart-stats]] receives it, before dropping nil points."+ [:merge+ ::series-config+ [:map {:closed true}+ [:x_values [:sequential [:maybe ::axis-value]]]+ [:y_values [:sequential [:maybe number?]]]]])+(mr/def ::timeline-event"A timeline event that may be relevant to the chart."[:map {:closed true}@@ -43,7 +51,7 @@(mr/def ::chart-config"Full chart configuration received from ai-service."[:map {:closed true}- [:series [:map-of :string ::series-config]]+ [:series [:map-of :string ::raw-series-config]][:timeline_events {:optional true} [:maybe [:sequential ::timeline-event]]][:query {:optional true} [:maybe ::lib-be.schema/maybe-legacy-query]][:display_type {:optional true} [:maybe :string]]diff --git a/src/metabase/interestingness/core.clj b/src/metabase/interestingness/core.cljindex c56b8a7e32d..74f3a2c8a24 100644--- a/src/metabase/interestingness/core.clj+++ b/src/metabase/interestingness/core.clj@@ -25,5 +25,6 @@generate-representationtemporal-context][metabase.interestingness.chart.stats+ analyzable-serieschart-stats-schemacompute-chart-stats])diff --git a/src/metabase/metabot/tools/analyze_chart.clj b/src/metabase/metabot/tools/analyze_chart.cljindex 72fa85aa22c..b49842af653 100644--- a/src/metabase/metabot/tools/analyze_chart.clj+++ b/src/metabase/metabot/tools/analyze_chart.clj@@ -57,14 +57,11 @@ Do not use headers (##). Do not list statistics. Do not analyze series separatel(defn- validate-chart-config!"Throw an informative exception when `chart-config` is too malformed to analyze."- [{:keys [series] :as _chart-config}]- (when (empty? series)+ [{:keys [series] :as chart-config}]+ (when (empty? (interestingness/analyzable-series chart-config))(throw (ex-info "This chart has no series data to analyze."{:agent-error? true, :type ::malformed-chart-config})))(doseq [[series-name {:keys [y_values]}] series]- (when (empty? y_values)- (throw (ex-info (format "Series \"%s\" has no data points to analyze." series-name)- {:agent-error? true, :type ::malformed-chart-config, :series series-name})))(when (non-numeric-y-value? y_values);; compute-chart-stats assumes each series has numeric y-values.(throw (ex-info (format (str "Series \"%s\" has non-numeric y-values. Chart analysis "diff --git a/test/metabase/interestingness/chart/stats_test.clj b/test/metabase/interestingness/chart/stats_test.cljindex 5e8c3c3fcc8..78b671e0f4f 100644--- a/test/metabase/interestingness/chart/stats_test.clj+++ b/test/metabase/interestingness/chart/stats_test.clj@@ -117,6 +117,52 @@max-pairs (/ (* max-k (dec max-k)) 2)](is (<= (count (:correlations stats)) max-pairs))))))+;;; ----------------------------------------------- Nil Values -----------------------------------------------------++(deftest ^:parallel time-series-skips-nil-y-values-test+ (testing "a period with a nil value is left out of the stats"+ (is (=? {:series [{:data-points 4+ :trend {:start-value 0.0 :end-value 4.0}}]}+ (stats.core/compute-chart-stats (assoc-in (make-chart-config 1 5) [:series "series_0" :y_values 2] nil)+ {:deep? true})))))++(deftest ^:parallel time-series-skips-nil-x-values-test+ (testing "a nil date bucket is not analyzed as a period"+ (is (=? {:series [{:data-points 4+ :time-range {:start "2020-01-02"}+ :trend {:start-value 1.0}}]}+ (stats.core/compute-chart-stats (assoc-in (make-chart-config 1 5) [:series "series_0" :x_values 0] nil)+ {:deep? true})))))++(deftest ^:parallel time-series-skips-series-with-only-nil-dates-test+ (testing "a series whose dates are all nil is left out, and the other series still gets stats"+ (let [config (assoc-in (make-chart-config 2 5) [:series "series_0" :x_values] [nil nil nil nil nil])]+ (is (=? {:series-count 1+ :series [{:name "series_1" :data-points 5}]}+ (stats.core/compute-chart-stats config {:deep? true}))))))++(deftest ^:parallel chart-type-ignores-dropped-series-test+ (testing "a first series whose categories are all nil does not pick the chart type"+ (let [counts (assoc (make-series 3) :x {:name "Category" :type "string"} :x_values ["Gadget" "Gizmo" "Widget"])]+ (doseq [display-type ["bar" "line"]]+ (testing display-type+ (is (=? {:chart-type :categorical+ :series-count 1+ :series [{:name "Globex"}]}+ (stats.core/compute-chart-stats {:display_type display-type+ :series {"Acme" (assoc counts :x_values [nil nil nil])+ "Globex" counts}}+ {:deep? true}))))))))++(deftest ^:parallel correlations-skip-nil-y-values-test+ (testing "a nil in one series leaves its other points to correlate"+ (let [series (assoc (make-series 12) :x {:name "Category" :type "string"} :x_values (mapv str (range 12)))]+ (is (=? {:correlations [{:strength :strong :direction :positive :aligned-sample-size 11}]}+ (stats.core/compute-chart-stats {:display_type "bar"+ :series {"a" (assoc-in series [:y_values 4] nil)+ "b" series}}+ {:deep? true}))))))+;;; ---------------------------------------- chart-stats-schema codec ----------------------------------------------(def ^:private codecdiff --git a/test/metabase/interestingness/chart_test.clj b/test/metabase/interestingness/chart_test.cljindex b3acf98c342..ec8f157212a 100644--- a/test/metabase/interestingness/chart_test.clj+++ b/test/metabase/interestingness/chart_test.clj@@ -85,6 +85,13 @@(is (nil? (:signal bd)))(is (nil? (:structure bd))))))+(deftest ^:parallel series-without-dates-is-not-scored-test+ (testing "a series whose dates are all nil scores as if it weren't there"+ (let [cfg (time-series-config (range 30))+ no-dates (assoc (get-in cfg [:series "s1"]) :x_values (vec (repeat 30 nil)))]+ (is (= (chart/chart-interestingness cfg)+ (chart/chart-interestingness (assoc-in cfg [:series "s2"] no-dates)))))))+(deftest ^:parallel caller-supplied-stats-test(testing "2-arity form accepts pre-computed stats (no recomputation)"(let [cfg (time-series-config (range 30))diff --git a/test/metabase/metabot/tools/analyze_chart_test.clj b/test/metabase/metabot/tools/analyze_chart_test.cljindex a6e009611c0..397ca92098e 100644--- a/test/metabase/metabot/tools/analyze_chart_test.clj+++ b/test/metabase/metabot/tools/analyze_chart_test.clj@@ -49,8 +49,8 @@(analyze "c1" {"c1" {:display_type "row" :title "Empty" :series {}}})))))(deftest ^:parallel analyze-chart-tool-empty-y-values-test- (testing "a series with no data points is rejected"- (is (= {:output "Series \"Count\" has no data points to analyze."}+ (testing "a chart whose only series has no data points is rejected"+ (is (= {:output "This chart has no series data to analyze."}(analyze "c1" {"c1" {:display_type "row":title "No points":series {"Count" {:x {:name "category" :type "string"}@@ -59,6 +59,35 @@:y_values []:display_name "Count"}}}})))))+(deftest ^:parallel analyze-chart-tool-no-complete-points-test+ (testing "a chart whose only series has no point with both an x and a y value is rejected like an empty one"+ (doseq [[x-values y-values] [[[nil nil] [3.0 4.0]]+ [["2024-01-01T00:00:00Z" "2024-02-01T00:00:00Z"] [nil nil]]+ [[nil "2024-01-01T00:00:00Z"] [3.0 nil]]]]+ (testing (pr-str x-values y-values)+ (is (= {:output "This chart has no series data to analyze."}+ (analyze "c1" {"c1" {:display_type "line"+ :title "Orders over time"+ :series {"Count" {:x {:name "created_at" :type "datetime"}+ :y {:name "count" :type "number"}+ :x_values x-values+ :y_values y-values+ :display_name "Count"}}}})))))))++(deftest ^:parallel analyze-chart-tool-all-null-series-test+ (testing "a series with only null values is left out, and the other series is analyzed"+ (let [series (fn [y-name y-values]+ {:x {:name "created_at" :type "datetime"}+ :y {:name y-name :type "number"}+ :x_values ["2024-01-01T00:00:00Z" "2024-02-01T00:00:00Z" "2024-03-01T00:00:00Z"]+ :y_values y-values+ :display_name y-name})]+ (is (str/includes? (:output (analyze "c1" {"c1" {:display_type "line"+ :title "Orders over time"+ :series {"Count" (series "Count" [3.0 4.0 5.0])+ "Refunds" (series "Refunds" [nil nil nil])}}}))+ "## Series: Count")))))+(deftest ^:parallel analyze-chart-tool-non-numeric-y-values-test(testing "a series with non-numeric y-values is rejected"(is (= {:output (str "Series \"Count\" has non-numeric y-values. "