The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix trend lines when aggregating by datetime (#65873)
Nathan Voxlanda7425eff43e7
diff --git a/src/metabase/analyze/fingerprint/insights.clj b/src/metabase/analyze/fingerprint/insights.cljindex 223389d7ccf..30a585942b5 100644--- a/src/metabase/analyze/fingerprint/insights.clj+++ b/src/metabase/analyze/fingerprint/insights.clj@@ -297,14 +297,28 @@(group-by (fn [{base-type :base_typeeffective-type :effective_typesemantic-type :semantic_type- unit :unit}]+ unit :unit+ source :source+ lib-source :lib/source+ lib-breakout? :lib/breakout?}](cond- (isa? semantic-type :Relation/*) :others- (u.date/truncate-units unit) :datetimes- (u.date/extract-units unit) :numbers- (isa? (or effective-type base-type) :type/Temporal) :datetimes- (isa? base-type :type/Number) :numbers- :else :others))))]+ (isa? semantic-type :Relation/*) :others++ ;; Only count datetime columns from breakouts/dimensions, not aggregations+ ;; Aggregations of datetime values (like max(created_at)) are computed values,+ ;; not datetime dimensions for the X-axis (#62069)+ (and+ (or (u.date/truncate-units unit)+ (isa? (or effective-type base-type) :type/Temporal))+ (or lib-breakout?+ (= source :breakout)+ (and (not= source :aggregation)+ (not (= lib-source :source/aggregations)))))+ :datetimes++ (u.date/extract-units unit) :numbers+ (isa? base-type :type/Number) :numbers+ :else :others))))](cond(timeseries? cols-by-type) (timeseries-insight cols-by-type)- :else (fingerprinters/constant-fingerprinter nil))))+ :else (fingerprinters/constant-fingerprinter nil))))diff --git a/test/metabase/analyze/fingerprint/insights_test.clj b/test/metabase/analyze/fingerprint/insights_test.cljindex 54f3d297313..daa03a0686e 100644--- a/test/metabase/analyze/fingerprint/insights_test.clj+++ b/test/metabase/analyze/fingerprint/insights_test.clj@@ -12,18 +12,18 @@(def ^:private cols [{:base_type :type/DateTime} {:base_type :type/Number}])(deftest last-value-test- (doseq [{:keys [rows expected]} [{:rows [["2014" 100]- ["2015" 200]- ["2016" nil]- [nil 300]- [nil nil]- ["2017" 700]]+ (doseq [{:keys [rows expected]} [{:rows [["2014" 100]+ ["2015" 200]+ ["2016" nil]+ [nil 300]+ [nil nil]+ ["2017" 700]]:expected 700}- {:rows [["2017" 700]]+ {:rows [["2017" 700]]:expected 700}- {:rows []+ {:rows []:expected nil}- {:rows [[nil nil]]+ {:rows [[nil nil]]:expected nil}]](testing (format "rows = %s" rows)(is (= expected@@ -109,30 +109,30 @@(deftest ^:parallel round-to-precision-test(are [exp figs n] (= exp(round-to-precision figs n))- 1.0 1 1.234- 1.2 2 1.234- 1.3 2 1.278- 1.3 2 1.251+ 1.0 1 1.234+ 1.2 2 1.234+ 1.3 2 1.278+ 1.3 2 1.25112300.0 3 12345.670.00321 3 0.003209817))(deftest timeseries-insight-test- (is (= [{:last-value 144,+ (is (= [{:last-value 144,:previous-value 179,- :last-change -0.19553072625698323,- :slope -7.671473413418271,- :offset 137234.92983406168,+ :last-change -0.19553072625698323,+ :slope -7.671473413418271,+ :offset 137234.92983406168,:best-fit [:* 1.56726E227 [:exp [:* -0.02899533549378612 :x]]],- :unit :day,- :col nil}- {:last-value 2525,+ :unit :day,+ :col nil}+ {:last-value 2525,:previous-value 3311,- :last-change -0.2373905164602839,- :slope -498.764272733624,- :offset 8915371.843617931,- :best-fit [:+ 8915371.843617931 [:* -498.764272733624 :x]],- :col nil,- :unit :day}]+ :last-change -0.2373905164602839,+ :slope -498.764272733624,+ :offset 8915371.843617931,+ :best-fit [:+ 8915371.843617931 [:* -498.764272733624 :x]],+ :col nil,+ :unit :day}](-> (transduce identity(insights/insights [{:base_type :type/DateTime}{:base_type :type/Number}@@ -141,14 +141,14 @@; This value varies between machines (M1 Macs? JVMs?) so round it to avoid test failures.(update-in [0 :best-fit 1] #(round-to-precision 6 %)))))(testing "We should robustly survive weird values such as NaN, Infinity, and nil"- (is (= [{:last-value 20.0+ (is (= [{:last-value 20.0:previous-value 10.0- :last-change 1.0- :slope 10.0- :offset -178350.0- :best-fit [:+ -178350.0 [:* 10.0 :x]]- :unit :day- :col nil}]+ :last-change 1.0+ :slope 10.0+ :offset -178350.0+ :best-fit [:+ -178350.0 [:* 10.0 :x]]+ :unit :day+ :col nil}](transduce identity(insights/insights [{:base_type :type/DateTime} {:base_type :type/Number}])[["2018-11-01" 10.0]@@ -210,11 +210,45 @@{:base_type :type/Number}])[["2024-08-09" 10.0]["2024-08-10" 20.0]]))- nil? {:base_type :type/Text}- some? {:base_type :type/Text :effective_type :type/DateTime}+ nil? {:base_type :type/Text}+ some? {:base_type :type/Text :effective_type :type/DateTime};; Extraction unit (day-of-week) is classified as a numeric column and doesn't produce insights here- nil? {:base_type :type/Text :unit :day-of-week}+ nil? {:base_type :type/Text :unit :day-of-week};; Spot check truncation units — should all generate insightssome? {:base_type :type/Text :unit :day}some? {:base_type :type/Text :unit :month}some? {:base_type :type/Text :unit :year})))++(deftest ^:parallel timeseries-with-datetime-aggregation-test+ (testing "A timeseries should still be detected when datetime aggregations are present (#62069)"+ (testing "Should recognize timeseries with datetime breakout and datetime aggregation (legacy :source)"+ (is (some?+ (transduce identity+ (insights/insights [{:base_type :type/DateTime+ :source :breakout}+ {:base_type :type/Number}+ {:base_type :type/DateTime+ :source :aggregation}])+ [["2024-08-09" 10.0 "2024-08-01"]+ ["2024-08-10" 20.0 "2024-08-02"]]))))+ (testing "Should recognize timeseries with lib/breakout? and lib/source metadata (modern)"+ (is (some?+ (transduce identity+ (insights/insights [{:base_type :type/DateTime+ :lib/breakout? true}+ {:base_type :type/Number}+ {:base_type :type/DateTime+ :lib/source :source/aggregations}])+ [["2024-08-09" 10.0 "2024-08-01"]+ ["2024-08-10" 20.0 "2024-08-02"]]))))+ (testing "Should work with effective_type for datetime aggregations"+ (is (some?+ (transduce identity+ (insights/insights [{:base_type :type/DateTime+ :source :breakout}+ {:base_type :type/Number}+ {:base_type :type/Text+ :effective_type :type/DateTime+ :source :aggregation}])+ [["2024-08-09" 10.0 "2024-08-01"]+ ["2024-08-10" 20.0 "2024-08-02"]]))))))