Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix trend lines when aggregating by datetime (#65873)

Nathan Voxlanda7425eff43e7
diff --git a/src/metabase/analyze/fingerprint/insights.clj b/src/metabase/analyze/fingerprint/insights.clj
index 223389d7ccf..30a585942b5 100644
--- a/src/metabase/analyze/fingerprint/insights.clj
+++ b/src/metabase/analyze/fingerprint/insights.clj
@@ -297,14 +297,28 @@
(group-by (fn [{base-type :base_type
effective-type :effective_type
semantic-type :semantic_type
- unit :unit}]
+ unit :unit
+ source :source
+ lib-source :lib/source
+ lib-breakout? :lib/breakout?}]
(cond
- (isa? semantic-type :Relation/*) :others
- (u.date/truncate-units unit) :datetimes
- (u.date/extract-units unit) :numbers
- (isa? (or effective-type base-type) :type/Temporal) :datetimes
- (isa? base-type :type/Number) :numbers
- :else :others))))]
+ (isa? semantic-type :Relation/*) :others
+
+ ;; Only count datetime columns from breakouts/dimensions, not aggregations
+ ;; Aggregations of datetime values (like max(created_at)) are computed values,
+ ;; not datetime dimensions for the X-axis (#62069)
+ (and
+ (or (u.date/truncate-units unit)
+ (isa? (or effective-type base-type) :type/Temporal))
+ (or lib-breakout?
+ (= source :breakout)
+ (and (not= source :aggregation)
+ (not (= lib-source :source/aggregations)))))
+ :datetimes
+
+ (u.date/extract-units unit) :numbers
+ (isa? base-type :type/Number) :numbers
+ :else :others))))]
(cond
(timeseries? cols-by-type) (timeseries-insight cols-by-type)
- :else (fingerprinters/constant-fingerprinter nil))))
+ :else (fingerprinters/constant-fingerprinter nil))))
diff --git a/test/metabase/analyze/fingerprint/insights_test.clj b/test/metabase/analyze/fingerprint/insights_test.clj
index 54f3d297313..daa03a0686e 100644
--- a/test/metabase/analyze/fingerprint/insights_test.clj
+++ b/test/metabase/analyze/fingerprint/insights_test.clj
@@ -12,18 +12,18 @@
(def ^:private cols [{:base_type :type/DateTime} {:base_type :type/Number}])
(deftest last-value-test
- (doseq [{:keys [rows expected]} [{:rows [["2014" 100]
- ["2015" 200]
- ["2016" nil]
- [nil 300]
- [nil nil]
- ["2017" 700]]
+ (doseq [{:keys [rows expected]} [{:rows [["2014" 100]
+ ["2015" 200]
+ ["2016" nil]
+ [nil 300]
+ [nil nil]
+ ["2017" 700]]
:expected 700}
- {:rows [["2017" 700]]
+ {:rows [["2017" 700]]
:expected 700}
- {:rows []
+ {:rows []
:expected nil}
- {:rows [[nil nil]]
+ {:rows [[nil nil]]
:expected nil}]]
(testing (format "rows = %s" rows)
(is (= expected
@@ -109,30 +109,30 @@
(deftest ^:parallel round-to-precision-test
(are [exp figs n] (= exp
(round-to-precision figs n))
- 1.0 1 1.234
- 1.2 2 1.234
- 1.3 2 1.278
- 1.3 2 1.251
+ 1.0 1 1.234
+ 1.2 2 1.234
+ 1.3 2 1.278
+ 1.3 2 1.251
12300.0 3 12345.67
0.00321 3 0.003209817))
(deftest timeseries-insight-test
- (is (= [{:last-value 144,
+ (is (= [{:last-value 144,
:previous-value 179,
- :last-change -0.19553072625698323,
- :slope -7.671473413418271,
- :offset 137234.92983406168,
+ :last-change -0.19553072625698323,
+ :slope -7.671473413418271,
+ :offset 137234.92983406168,
:best-fit [:* 1.56726E227 [:exp [:* -0.02899533549378612 :x]]],
- :unit :day,
- :col nil}
- {:last-value 2525,
+ :unit :day,
+ :col nil}
+ {:last-value 2525,
:previous-value 3311,
- :last-change -0.2373905164602839,
- :slope -498.764272733624,
- :offset 8915371.843617931,
- :best-fit [:+ 8915371.843617931 [:* -498.764272733624 :x]],
- :col nil,
- :unit :day}]
+ :last-change -0.2373905164602839,
+ :slope -498.764272733624,
+ :offset 8915371.843617931,
+ :best-fit [:+ 8915371.843617931 [:* -498.764272733624 :x]],
+ :col nil,
+ :unit :day}]
(-> (transduce identity
(insights/insights [{:base_type :type/DateTime}
{:base_type :type/Number}
@@ -141,14 +141,14 @@
; This value varies between machines (M1 Macs? JVMs?) so round it to avoid test failures.
(update-in [0 :best-fit 1] #(round-to-precision 6 %)))))
(testing "We should robustly survive weird values such as NaN, Infinity, and nil"
- (is (= [{:last-value 20.0
+ (is (= [{:last-value 20.0
:previous-value 10.0
- :last-change 1.0
- :slope 10.0
- :offset -178350.0
- :best-fit [:+ -178350.0 [:* 10.0 :x]]
- :unit :day
- :col nil}]
+ :last-change 1.0
+ :slope 10.0
+ :offset -178350.0
+ :best-fit [:+ -178350.0 [:* 10.0 :x]]
+ :unit :day
+ :col nil}]
(transduce identity
(insights/insights [{:base_type :type/DateTime} {:base_type :type/Number}])
[["2018-11-01" 10.0]
@@ -210,11 +210,45 @@
{:base_type :type/Number}])
[["2024-08-09" 10.0]
["2024-08-10" 20.0]]))
- nil? {:base_type :type/Text}
- some? {:base_type :type/Text :effective_type :type/DateTime}
+ nil? {:base_type :type/Text}
+ some? {:base_type :type/Text :effective_type :type/DateTime}
;; Extraction unit (day-of-week) is classified as a numeric column and doesn't produce insights here
- nil? {:base_type :type/Text :unit :day-of-week}
+ nil? {:base_type :type/Text :unit :day-of-week}
;; Spot check truncation units — should all generate insights
some? {:base_type :type/Text :unit :day}
some? {:base_type :type/Text :unit :month}
some? {:base_type :type/Text :unit :year})))
+
+(deftest ^:parallel timeseries-with-datetime-aggregation-test
+ (testing "A timeseries should still be detected when datetime aggregations are present (#62069)"
+ (testing "Should recognize timeseries with datetime breakout and datetime aggregation (legacy :source)"
+ (is (some?
+ (transduce identity
+ (insights/insights [{:base_type :type/DateTime
+ :source :breakout}
+ {:base_type :type/Number}
+ {:base_type :type/DateTime
+ :source :aggregation}])
+ [["2024-08-09" 10.0 "2024-08-01"]
+ ["2024-08-10" 20.0 "2024-08-02"]]))))
+ (testing "Should recognize timeseries with lib/breakout? and lib/source metadata (modern)"
+ (is (some?
+ (transduce identity
+ (insights/insights [{:base_type :type/DateTime
+ :lib/breakout? true}
+ {:base_type :type/Number}
+ {:base_type :type/DateTime
+ :lib/source :source/aggregations}])
+ [["2024-08-09" 10.0 "2024-08-01"]
+ ["2024-08-10" 20.0 "2024-08-02"]]))))
+ (testing "Should work with effective_type for datetime aggregations"
+ (is (some?
+ (transduce identity
+ (insights/insights [{:base_type :type/DateTime
+ :source :breakout}
+ {:base_type :type/Number}
+ {:base_type :type/Text
+ :effective_type :type/DateTime
+ :source :aggregation}])
+ [["2024-08-09" 10.0 "2024-08-01"]
+ ["2024-08-10" 20.0 "2024-08-02"]]))))))