Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Skip null values when analyzing a chart (#84261)

Andrei Simionescua1ca09fcfd9e
diff --git a/src/metabase/interestingness/chart.clj b/src/metabase/interestingness/chart.clj
index ad2d40b4855..be9a7cf86fb 100644
--- a/src/metabase/interestingness/chart.clj
+++ b/src/metabase/interestingness/chart.clj
@@ -168,8 +168,9 @@
(chart-interestingness chart-config
(chart.stats/compute-chart-stats chart-config {})))
([chart-config stats]
- (let [nd (non-degeneracy-score chart-config)
- chart-type (:chart-type stats)]
+ (let [chart-config (assoc chart-config :series (chart.stats/analyzable-series chart-config))
+ nd (non-degeneracy-score chart-config)
+ chart-type (:chart-type stats)]
(if (zero? nd)
{:score 0.0 :non-degeneracy nd :signal nil :structure nil :chart-type chart-type}
(let [sig (signal-score stats)
diff --git a/src/metabase/interestingness/chart/stats.clj b/src/metabase/interestingness/chart/stats.clj
index b19f3db8b83..0dc8c19d16d 100644
--- a/src/metabase/interestingness/chart/stats.clj
+++ b/src/metabase/interestingness/chart/stats.clj
@@ -1,6 +1,7 @@
(ns metabase.interestingness.chart.stats
"Chart type detection and statistics routing."
(:require
+ [medley.core :as m]
[metabase.interestingness.chart.categorical :as categorical]
[metabase.interestingness.chart.histogram :as histogram]
[metabase.interestingness.chart.scatter :as scatter]
@@ -56,6 +57,20 @@
(when (seq downsampled)
{:downsampled-series downsampled})]))
+(defn- drop-nil-points
+ "Drop the points of `series-config` whose x or y value is nil."
+ [{x-values :x_values, y-values :y_values, :as series-config}]
+ (let [points (filter (fn [[x y]] (and (some? x) (some? y))) (map vector x-values y-values))]
+ (assoc series-config
+ :x_values (mapv first points)
+ :y_values (mapv second points))))
+
+(defn analyzable-series
+ "Return the series of `chart-config` with their nil points dropped, leaving out any series with no points left."
+ [chart-config]
+ (->> (update-vals (:series chart-config) drop-nil-points)
+ (m/filter-vals (comp seq :y_values))))
+
;;; ----------------------------------------------- Chart Type Detection ---------------------------------------------
(def ^:private explicit-display-types
@@ -138,7 +153,7 @@
(mu/defn compute-chart-stats :- ::stats.types/chart-stats
"Compute statistics for a chart based on its detected type.
- Applies data limits before computation:
+ Drops points with a nil x or y value and series left empty, then applies data limits before computation:
- Each series is downsampled to at most [[max-data-points-per-series]] points
- Correlation computation is limited to [[max-series-for-correlations]] series
@@ -148,8 +163,9 @@
:deep? - compute additional statistics"
[chart-config :- ::stats.types/chart-config
opts :- ::stats.types/options]
- (let [chart-type (detect-chart-type chart-config)
- [limited-series limits-info] (apply-data-limits (:series chart-config))
+ (let [series (analyzable-series chart-config)
+ chart-type (detect-chart-type (assoc chart-config :series series))
+ [limited-series limits-info] (apply-data-limits series)
too-many-series? (> (count limited-series) max-series-for-correlations)
opts (cond-> opts
too-many-series? (assoc :max-correlation-series max-series-for-correlations))
diff --git a/src/metabase/interestingness/chart/types.clj b/src/metabase/interestingness/chart/types.clj
index 99a7c45f8c1..d82e46e3174 100644
--- a/src/metabase/interestingness/chart/types.clj
+++ b/src/metabase/interestingness/chart/types.clj
@@ -32,6 +32,14 @@
[:chart_type {:optional true} [:maybe :string]]
[:stacked {:optional true} [:maybe :boolean]]])
+(mr/def ::raw-series-config
+ "A series as [[metabase.interestingness.chart.stats/compute-chart-stats]] receives it, before dropping nil points."
+ [:merge
+ ::series-config
+ [:map {:closed true}
+ [:x_values [:sequential [:maybe ::axis-value]]]
+ [:y_values [:sequential [:maybe number?]]]]])
+
(mr/def ::timeline-event
"A timeline event that may be relevant to the chart."
[:map {:closed true}
@@ -43,7 +51,7 @@
(mr/def ::chart-config
"Full chart configuration received from ai-service."
[:map {:closed true}
- [:series [:map-of :string ::series-config]]
+ [:series [:map-of :string ::raw-series-config]]
[:timeline_events {:optional true} [:maybe [:sequential ::timeline-event]]]
[:query {:optional true} [:maybe ::lib-be.schema/maybe-legacy-query]]
[:display_type {:optional true} [:maybe :string]]
diff --git a/src/metabase/interestingness/core.clj b/src/metabase/interestingness/core.clj
index c56b8a7e32d..74f3a2c8a24 100644
--- a/src/metabase/interestingness/core.clj
+++ b/src/metabase/interestingness/core.clj
@@ -25,5 +25,6 @@
generate-representation
temporal-context]
[metabase.interestingness.chart.stats
+ analyzable-series
chart-stats-schema
compute-chart-stats])
diff --git a/src/metabase/metabot/tools/analyze_chart.clj b/src/metabase/metabot/tools/analyze_chart.clj
index 72fa85aa22c..b49842af653 100644
--- a/src/metabase/metabot/tools/analyze_chart.clj
+++ b/src/metabase/metabot/tools/analyze_chart.clj
@@ -57,14 +57,11 @@ Do not use headers (##). Do not list statistics. Do not analyze series separatel
(defn- validate-chart-config!
"Throw an informative exception when `chart-config` is too malformed to analyze."
- [{:keys [series] :as _chart-config}]
- (when (empty? series)
+ [{:keys [series] :as chart-config}]
+ (when (empty? (interestingness/analyzable-series chart-config))
(throw (ex-info "This chart has no series data to analyze."
{:agent-error? true, :type ::malformed-chart-config})))
(doseq [[series-name {:keys [y_values]}] series]
- (when (empty? y_values)
- (throw (ex-info (format "Series \"%s\" has no data points to analyze." series-name)
- {:agent-error? true, :type ::malformed-chart-config, :series series-name})))
(when (non-numeric-y-value? y_values)
;; compute-chart-stats assumes each series has numeric y-values.
(throw (ex-info (format (str "Series \"%s\" has non-numeric y-values. Chart analysis "
diff --git a/test/metabase/interestingness/chart/stats_test.clj b/test/metabase/interestingness/chart/stats_test.clj
index 5e8c3c3fcc8..78b671e0f4f 100644
--- a/test/metabase/interestingness/chart/stats_test.clj
+++ b/test/metabase/interestingness/chart/stats_test.clj
@@ -117,6 +117,52 @@
max-pairs (/ (* max-k (dec max-k)) 2)]
(is (<= (count (:correlations stats)) max-pairs))))))
+;;; ----------------------------------------------- Nil Values -----------------------------------------------------
+
+(deftest ^:parallel time-series-skips-nil-y-values-test
+ (testing "a period with a nil value is left out of the stats"
+ (is (=? {:series [{:data-points 4
+ :trend {:start-value 0.0 :end-value 4.0}}]}
+ (stats.core/compute-chart-stats (assoc-in (make-chart-config 1 5) [:series "series_0" :y_values 2] nil)
+ {:deep? true})))))
+
+(deftest ^:parallel time-series-skips-nil-x-values-test
+ (testing "a nil date bucket is not analyzed as a period"
+ (is (=? {:series [{:data-points 4
+ :time-range {:start "2020-01-02"}
+ :trend {:start-value 1.0}}]}
+ (stats.core/compute-chart-stats (assoc-in (make-chart-config 1 5) [:series "series_0" :x_values 0] nil)
+ {:deep? true})))))
+
+(deftest ^:parallel time-series-skips-series-with-only-nil-dates-test
+ (testing "a series whose dates are all nil is left out, and the other series still gets stats"
+ (let [config (assoc-in (make-chart-config 2 5) [:series "series_0" :x_values] [nil nil nil nil nil])]
+ (is (=? {:series-count 1
+ :series [{:name "series_1" :data-points 5}]}
+ (stats.core/compute-chart-stats config {:deep? true}))))))
+
+(deftest ^:parallel chart-type-ignores-dropped-series-test
+ (testing "a first series whose categories are all nil does not pick the chart type"
+ (let [counts (assoc (make-series 3) :x {:name "Category" :type "string"} :x_values ["Gadget" "Gizmo" "Widget"])]
+ (doseq [display-type ["bar" "line"]]
+ (testing display-type
+ (is (=? {:chart-type :categorical
+ :series-count 1
+ :series [{:name "Globex"}]}
+ (stats.core/compute-chart-stats {:display_type display-type
+ :series {"Acme" (assoc counts :x_values [nil nil nil])
+ "Globex" counts}}
+ {:deep? true}))))))))
+
+(deftest ^:parallel correlations-skip-nil-y-values-test
+ (testing "a nil in one series leaves its other points to correlate"
+ (let [series (assoc (make-series 12) :x {:name "Category" :type "string"} :x_values (mapv str (range 12)))]
+ (is (=? {:correlations [{:strength :strong :direction :positive :aligned-sample-size 11}]}
+ (stats.core/compute-chart-stats {:display_type "bar"
+ :series {"a" (assoc-in series [:y_values 4] nil)
+ "b" series}}
+ {:deep? true}))))))
+
;;; ---------------------------------------- chart-stats-schema codec ----------------------------------------------
(def ^:private codec
diff --git a/test/metabase/interestingness/chart_test.clj b/test/metabase/interestingness/chart_test.clj
index b3acf98c342..ec8f157212a 100644
--- a/test/metabase/interestingness/chart_test.clj
+++ b/test/metabase/interestingness/chart_test.clj
@@ -85,6 +85,13 @@
(is (nil? (:signal bd)))
(is (nil? (:structure bd))))))
+(deftest ^:parallel series-without-dates-is-not-scored-test
+ (testing "a series whose dates are all nil scores as if it weren't there"
+ (let [cfg (time-series-config (range 30))
+ no-dates (assoc (get-in cfg [:series "s1"]) :x_values (vec (repeat 30 nil)))]
+ (is (= (chart/chart-interestingness cfg)
+ (chart/chart-interestingness (assoc-in cfg [:series "s2"] no-dates)))))))
+
(deftest ^:parallel caller-supplied-stats-test
(testing "2-arity form accepts pre-computed stats (no recomputation)"
(let [cfg (time-series-config (range 30))
diff --git a/test/metabase/metabot/tools/analyze_chart_test.clj b/test/metabase/metabot/tools/analyze_chart_test.clj
index a6e009611c0..397ca92098e 100644
--- a/test/metabase/metabot/tools/analyze_chart_test.clj
+++ b/test/metabase/metabot/tools/analyze_chart_test.clj
@@ -49,8 +49,8 @@
(analyze "c1" {"c1" {:display_type "row" :title "Empty" :series {}}})))))
(deftest ^:parallel analyze-chart-tool-empty-y-values-test
- (testing "a series with no data points is rejected"
- (is (= {:output "Series \"Count\" has no data points to analyze."}
+ (testing "a chart whose only series has no data points is rejected"
+ (is (= {:output "This chart has no series data to analyze."}
(analyze "c1" {"c1" {:display_type "row"
:title "No points"
:series {"Count" {:x {:name "category" :type "string"}
@@ -59,6 +59,35 @@
:y_values []
:display_name "Count"}}}})))))
+(deftest ^:parallel analyze-chart-tool-no-complete-points-test
+ (testing "a chart whose only series has no point with both an x and a y value is rejected like an empty one"
+ (doseq [[x-values y-values] [[[nil nil] [3.0 4.0]]
+ [["2024-01-01T00:00:00Z" "2024-02-01T00:00:00Z"] [nil nil]]
+ [[nil "2024-01-01T00:00:00Z"] [3.0 nil]]]]
+ (testing (pr-str x-values y-values)
+ (is (= {:output "This chart has no series data to analyze."}
+ (analyze "c1" {"c1" {:display_type "line"
+ :title "Orders over time"
+ :series {"Count" {:x {:name "created_at" :type "datetime"}
+ :y {:name "count" :type "number"}
+ :x_values x-values
+ :y_values y-values
+ :display_name "Count"}}}})))))))
+
+(deftest ^:parallel analyze-chart-tool-all-null-series-test
+ (testing "a series with only null values is left out, and the other series is analyzed"
+ (let [series (fn [y-name y-values]
+ {:x {:name "created_at" :type "datetime"}
+ :y {:name y-name :type "number"}
+ :x_values ["2024-01-01T00:00:00Z" "2024-02-01T00:00:00Z" "2024-03-01T00:00:00Z"]
+ :y_values y-values
+ :display_name y-name})]
+ (is (str/includes? (:output (analyze "c1" {"c1" {:display_type "line"
+ :title "Orders over time"
+ :series {"Count" (series "Count" [3.0 4.0 5.0])
+ "Refunds" (series "Refunds" [nil nil nil])}}}))
+ "## Series: Count")))))
+
(deftest ^:parallel analyze-chart-tool-non-numeric-y-values-test
(testing "a series with non-numeric y-values is rejected"
(is (= {:output (str "Series \"Count\" has non-numeric y-values. "