Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Replace bigml Histogram and stream-lib with DataSketches (#66642)

Oleksandr Yakusheve979f409973f
diff --git a/bin/build/resources/overrides.edn b/bin/build/resources/overrides.edn
index ff1f2740589..38887b30863 100644
--- a/bin/build/resources/overrides.edn
+++ b/bin/build/resources/overrides.edn
@@ -10,6 +10,7 @@
"com.onelogin" {:resource "MIT.txt"}
"com.vladsch.flexmark" {:resource "BSD.txt"}
"io.netty" {:resource "apache2_0.txt"}
+ "org.apache.datasketches" {:resource "apache2_0.txt"}
"org.apache.hadoop" {:resource "apache2_0.txt"}
"org.clojure" {:resource "EPL.txt"}
"org.eclipse.jetty" {:resource "apache2_0.txt"}
diff --git a/deps.edn b/deps.edn
index b67e71cf478..314b7ca1844 100644
--- a/deps.edn
+++ b/deps.edn
@@ -6,8 +6,6 @@
;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
{amalloy/ring-gzip-middleware {:mvn/version "0.1.4"} ; Ring middleware to GZIP responses if client can handle it
babashka/fs {:mvn/version "0.5.27"} ; Portable filesystem operations
- bigml/histogram {:mvn/version "5.0.0" ; Histogram data structure
- :exclusions [junit/junit]}
buddy/buddy-core {:mvn/version "1.12.0-430" ; various cryptographic functions
:exclusions [commons-codec/commons-codec
org.bouncycastle/bcpkix-jdk15on
@@ -30,9 +28,6 @@
com.zaxxer/HikariCP-java7]}
colorize/colorize {:mvn/version "0.1.1" ; string output with ANSI color codes (for logging)
:exclusions [org.clojure/clojure]}
- com.clearspring.analytics/stream {:mvn/version "2.9.8" ; Various sketching algorithms
- :exclusions [it.unimi.dsi/fastutil
- org.slf4j/slf4j-api]}
com.clojure-goes-fast/jvm-alloc-rate-meter {:mvn/version "0.1.4"} ; Monitor heap allocation rate.
com.clojure-goes-fast/jvm-hiccup-meter {:mvn/version "0.1.1"} ; Monitor GC pauses and other system-induced pauses.
com.draines/postal {:mvn/version "2.0.5" ; SMTP library
@@ -116,6 +111,8 @@
net.thisptr/jackson-jq {:mvn/version "1.6.0"} ; Java implementation of the JQ json query language
org.apache.commons/commons-compress {:mvn/version "1.28.0"} ; compression utils
org.apache.commons/commons-lang3 {:mvn/version "3.20.0"} ; helper methods for working with java.lang stuff
+ org.apache.datasketches/datasketches-java ^:antq/exclude ; Sketching algorithms (used for fingerprinting)
+ {:mvn/version "7.0.1"} ; version is fixed to ensure JDK21 compatibility.
org.apache.httpcomponents.client5/httpclient5 {:mvn/version "5.4.4"} ; pinned: ClickHouse JDBC 0.9.7 needs httpcore5 5.3.4+; must pair with httpclient5 5.4.x
org.apache.httpcomponents.core5/httpcore5 {:mvn/version "5.3.5"} ; pinned: ClickHouse JDBC 0.9.7 requires URIBuilder.optimize() added in 5.3
org.apache.httpcomponents.core5/httpcore5-h2 {:mvn/version "5.3.5"} ; pinned: must match httpcore5 version
diff --git a/src/metabase/analyze/fingerprint/fingerprinters.clj b/src/metabase/analyze/fingerprint/fingerprinters.clj
index 5f08cc765f9..4752e961bf0 100644
--- a/src/metabase/analyze/fingerprint/fingerprinters.clj
+++ b/src/metabase/analyze/fingerprint/fingerprinters.clj
@@ -1,7 +1,6 @@
(ns metabase.analyze.fingerprint.fingerprinters
"Non-identifying fingerprinters for various field types."
(:require
- [bigml.histogram.core :as hist]
[clojure.string :as str]
[java-time.api :as t]
[kixi.stats.core :as stats]
@@ -15,11 +14,13 @@
[metabase.util.performance :as perf]
[redux.core :as redux])
(:import
- (com.bigml.histogram Histogram)
- (com.clearspring.analytics.stream.cardinality HyperLogLogPlus)
- (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZoneOffset ZonedDateTime)
+ (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime ZoneOffset)
(java.time.chrono ChronoLocalDateTime ChronoZonedDateTime)
- (java.time.temporal ChronoField Temporal)))
+ (java.time.temporal ChronoField Temporal)
+ (org.apache.commons.codec.digest MurmurHash2)
+ (org.apache.commons.math3.stat.descriptive SummaryStatistics)
+ (org.apache.datasketches.hll HllSketch)
+ (org.apache.datasketches.kll KllDoublesSketch)))
(set! *warn-on-reflection* true)
@@ -51,13 +52,17 @@
([_ _] (reduced init))))
(defn- cardinality
- "Transducer that sketches cardinality using HyperLogLog++.
- https://research.google.com/pubs/pub40671.html"
- ([] (HyperLogLogPlus. 14 25))
- ([^HyperLogLogPlus acc] (.cardinality acc))
- ([^HyperLogLogPlus acc x]
- (.offer acc x)
- acc))
+ "Transducer that sketches cardinality using DataSketches' HyperLogLog implementation."
+ ([] (HllSketch. 16))
+ ([^HllSketch acc] (Math/round (.getEstimate acc)))
+ ([^HllSketch acc x]
+ ;; Hashing is implemented in this way to ensure better overlap with results that the previously used
+ ;; HyperLogLogPlus implementation produced.
+ (let [h (cond (string? x) (MurmurHash2/hash64 ^String x)
+ (bytes? x) (MurmurHash2/hash64 ^bytes x (alength ^bytes x))
+ :else (hash x))]
+ (.update acc ^long h)
+ acc)))
(defmacro robust-map
"Wrap each map value in try-catch block."
@@ -347,11 +352,17 @@
(assoc :mode-fraction (:mode-fraction mode-stats)
:top-3-fraction (:top-3-fraction mode-stats)))))))
-(defn- histogram
- "Transducer that summarizes numerical data with a histogram."
- ([] (hist/create))
- ([^Histogram histogram] histogram)
- ([^Histogram histogram x] (hist/insert-simple! histogram x)))
+(deftype ^:private DistributionHolder [^SummaryStatistics summary, ^KllDoublesSketch kll])
+
+(defn- distribution
+ "Transducer that summarizes numerical data with SummaryStatistics and KllDoublesSketch."
+ ([] (DistributionHolder. (SummaryStatistics.) (KllDoublesSketch/newHeapInstance)))
+ ([^DistributionHolder holder] holder)
+ ([^DistributionHolder holder x]
+ (let [d (double x)]
+ (.addValue ^SummaryStatistics (.summary holder) d)
+ (.update ^KllDoublesSketch (.kll holder) d)
+ holder)))
(defprotocol ^:private INumberCoerceable
"Protocol for converting objects to a java.lang.Number."
@@ -370,16 +381,19 @@
(deffingerprinter :type/Number
(redux/post-complete
((comp (map ->number) (filter u/real-number?))
- (redux/juxt histogram stats/skewness stats/kurtosis mode-stats (stats/share zero?)))
- (fn [[h sk ku ms zf]]
- (let [{q1 0.25 q3 0.75} (hist/percentiles h 0.25 0.75)]
+ (redux/juxt distribution stats/skewness stats/kurtosis mode-stats (stats/share zero?)))
+ (fn [[^DistributionHolder h, sk ku ms zf]]
+ (let [^SummaryStatistics summary (.summary h)
+ ^KllDoublesSketch kll (.kll h)
+ n (.getN summary)]
(robust-map
- :min (hist/minimum h)
- :max (hist/maximum h)
- :avg (hist/mean h)
- :sd (some-> h hist/variance math/sqrt)
- :q1 q1
- :q3 q3
+ :min (.getMinItem kll)
+ :max (.getMaxItem kll)
+ ;; Ensure we don't get ##NaN in avg/sd
+ :avg (when (pos? n) (.getMean summary))
+ :sd (when (pos? n) (math/sqrt (.getVariance summary)))
+ :q1 (.getQuantile kll 0.25)
+ :q3 (.getQuantile kll 0.75)
:skewness sk
:excess-kurtosis ku
:mode-fraction (:mode-fraction ms)
diff --git a/test/metabase/analyze/fingerprint/fingerprinters_test.clj b/test/metabase/analyze/fingerprint/fingerprinters_test.clj
index e93e9f21f98..a4eccc84611 100644
--- a/test/metabase/analyze/fingerprint/fingerprinters_test.clj
+++ b/test/metabase/analyze/fingerprint/fingerprinters_test.clj
@@ -114,28 +114,28 @@
[#t "2013" #t "2018" #t "2015"])))))))
(deftest ^:parallel fingerprint-numeric-values-test
- (is (= {:global {:distinct-count 3
+ (is (= {:global {:distinct-count 99
:nil% 0.0}
- :type {:type/Number {:avg 2.0
- :min 1.0
- :max 3.0
- :q1 1.25
- :q3 2.75
- :sd 1.0
+ :type {:type/Number {:avg 49.0
+ :min 0.0
+ :max 98.0
+ :q1 24.0
+ :q3 74.0
+ :sd 28.722813232690143
:skewness 0.0
- :excess-kurtosis nil
- :mode-fraction (/ 1.0 3.0)
- :top-3-fraction 1.0
- :zero-fraction 0.0}}}
+ :excess-kurtosis -1.2000000000000002
+ :mode-fraction 0.010101010101010102
+ :top-3-fraction 0.030303030303030304
+ :zero-fraction 0.0101010101010101}}}
(transduce identity
(fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Number}))
- [1.0 2.0 3.0])))
+ (map double (range 99)))))
(testing "we respect effective_type"
(let [result (transduce identity
(fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text :effective_type :type/Number}))
["1" "2" "1.3" "2.3"])]
(is (= {:distinct-count 4, :nil% 0.0} (:global result)))
- (is (= {:min 1.0 :q1 1.15 :q3 2.15 :max 2.3 :avg 1.65}
+ (is (= {:min 1.0 :q1 1.0 :q3 2.0 :max 2.3 :avg 1.65}
(select-keys (get-in result [:type :type/Number])
[:min :q1 :q3 :max :avg])))
(is (some? (get-in result [:type :type/Number :skewness])))
@@ -145,9 +145,26 @@
(fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Number}))
[1.0 2.0 3.0 Double/NaN Double/POSITIVE_INFINITY Double/NEGATIVE_INFINITY nil nil])]
(is (= {:distinct-count 7, :nil% 0.25} (:global result)))
- (is (= {:avg 2.0, :min 1.0, :max 3.0, :q1 1.25, :q3 2.75, :sd 1.0}
+ (is (= {:avg 2.0, :min 1.0, :max 3.0, :q1 1.0, :q3 3.0, :sd 1.0}
(select-keys (get-in result [:type :type/Number])
- [:avg :min :max :q1 :q3 :sd]))))))
+ [:avg :min :max :q1 :q3 :sd])))
+ (is (=? {:global {:distinct-count 4, :nil% 0.0},
+ :type {:type/Number {:min 1.0, :q1 1.0, :q3 2.0, :max 2.3, :sd 0.6027713773341707, :avg 1.65}}}
+ (transduce identity
+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text :effective_type :type/Number}))
+ ["1" "2" "1.3" "2.3"])))))
+ (testing "We should robustly survive weird values such as NaN, Infinity, and nil"
+ (is (=? {:global {:distinct-count 7
+ :nil% 0.25}
+ :type {:type/Number {:avg 2.0
+ :min 1.0
+ :max 3.0
+ :q1 1.0
+ :q3 3.0
+ :sd 1.0}}}
+ (transduce identity
+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Number}))
+ [1.0 2.0 3.0 Double/NaN Double/POSITIVE_INFINITY Double/NEGATIVE_INFINITY nil nil])))))
(deftest ^:parallel fingerprint-string-values-test
(is (= {:global {:distinct-count 5
@@ -214,11 +231,11 @@
(is (=? {:global {:distinct-count 4
:nil% 0.0}
:type {:type/Number {:min 1.0
- :q1 #(< 1.44 % 1.46)
- :q3 #(< 2.4 % 2.5)
+ :q1 2.0
+ :q3 2.0
:max 4.0
:sd #(< 0.76 % 0.78)
- :avg 2.03}}}
+ :avg #(< 2.02 % 2.04)}}}
(t2/select-one-fn :fingerprint :model/Field :id (mt/id :venues :price))))))))
(deftest ^:parallel valid-serialized-json?-test
diff --git a/test/metabase/analyze/query_results_test.clj b/test/metabase/analyze/query_results_test.clj
index c2ee79e9389..6905cda7e6e 100644
--- a/test/metabase/analyze/query_results_test.clj
+++ b/test/metabase/analyze/query_results_test.clj
@@ -85,7 +85,7 @@
(mt/with-temp [:model/Card card {:dataset_query {:database (mt/id), :type :native, :native {:query "select * from venues"}}}]
(is (=? (assoc-in (mt/round-all-decimals 2 (app-db-venue-fingerprints))
[:category_id :type]
- #:type{:Number {:min 2.0, :max 74.0, :avg 29.98, :q1 6.9, :q3 49.24, :sd 23.06}})
+ #:type{:Number {:min 2.0, :max 74.0, :avg 29.98, :q1 7.0, :q3 49.0, :sd 23.06}})
(->> (name->fingerprints (query->result-metadata (query-for-card card)))
(mt/round-all-decimals 2)))))))
diff --git a/test/metabase/lib/test_metadata.cljc b/test/metabase/lib/test_metadata.cljc
index 860da57a9b1..1eac9c8e4ca 100644
--- a/test/metabase/lib/test_metadata.cljc
+++ b/test/metabase/lib/test_metadata.cljc
@@ -808,8 +808,8 @@
:fingerprint {:global {:distinct-count 4, :nil% 0.0}
:type {:type/Number
{:min 1.0
- :q1 1.4591129021415095
- :q3 2.493086095768049
+ :q1 2.0
+ :q3 2.0
:max 4.0
:sd 0.7713951678941896
:avg 2.03}}}
diff --git a/test/metabase/query_processor/middleware/results_metadata_test.clj b/test/metabase/query_processor/middleware/results_metadata_test.clj
index 39740e967de..a1e651fed4c 100644
--- a/test/metabase/query_processor/middleware/results_metadata_test.clj
+++ b/test/metabase/query_processor/middleware/results_metadata_test.clj
@@ -90,8 +90,8 @@
(update-in [3 :fingerprint] assoc :type {:type/Number {:min 2.0
:max 74.0
:avg 29.98
- :q1 6.9
- :q3 49.24
+ :q1 7.0
+ :q3 49.0
:sd 23.06}}))]
(assoc column :display_name (:name column))))
@@ -290,7 +290,7 @@
:semantic_type :type/Quantity
:fingerprint {:global {:distinct-count 3
:nil% 0.0}
- :type {:type/Number {:min 235.0, :max 498.0, :avg 333.33 :q1 243.0, :q3 440.25, :sd 143.5}}}
+ :type {:type/Number {:min 235.0, :max 498.0, :avg 333.33 :q1 235.0, :q3 498.0, :sd 143.5}}}
:field_ref [:aggregation 0]}]
(-> card
card-metadata