The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Replace bigml Histogram and stream-lib with DataSketches (#66642)
Oleksandr Yakusheve979f409973f
diff --git a/bin/build/resources/overrides.edn b/bin/build/resources/overrides.ednindex ff1f2740589..38887b30863 100644--- a/bin/build/resources/overrides.edn+++ b/bin/build/resources/overrides.edn@@ -10,6 +10,7 @@"com.onelogin" {:resource "MIT.txt"}"com.vladsch.flexmark" {:resource "BSD.txt"}"io.netty" {:resource "apache2_0.txt"}+ "org.apache.datasketches" {:resource "apache2_0.txt"}"org.apache.hadoop" {:resource "apache2_0.txt"}"org.clojure" {:resource "EPL.txt"}"org.eclipse.jetty" {:resource "apache2_0.txt"}diff --git a/deps.edn b/deps.ednindex b67e71cf478..314b7ca1844 100644--- a/deps.edn+++ b/deps.edn@@ -6,8 +6,6 @@;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!{amalloy/ring-gzip-middleware {:mvn/version "0.1.4"} ; Ring middleware to GZIP responses if client can handle itbabashka/fs {:mvn/version "0.5.27"} ; Portable filesystem operations- bigml/histogram {:mvn/version "5.0.0" ; Histogram data structure- :exclusions [junit/junit]}buddy/buddy-core {:mvn/version "1.12.0-430" ; various cryptographic functions:exclusions [commons-codec/commons-codecorg.bouncycastle/bcpkix-jdk15on@@ -30,9 +28,6 @@com.zaxxer/HikariCP-java7]}colorize/colorize {:mvn/version "0.1.1" ; string output with ANSI color codes (for logging):exclusions [org.clojure/clojure]}- com.clearspring.analytics/stream {:mvn/version "2.9.8" ; Various sketching algorithms- :exclusions [it.unimi.dsi/fastutil- org.slf4j/slf4j-api]}com.clojure-goes-fast/jvm-alloc-rate-meter {:mvn/version "0.1.4"} ; Monitor heap allocation rate.com.clojure-goes-fast/jvm-hiccup-meter {:mvn/version "0.1.1"} ; Monitor GC pauses and other system-induced pauses.com.draines/postal {:mvn/version "2.0.5" ; SMTP library@@ -116,6 +111,8 @@net.thisptr/jackson-jq {:mvn/version "1.6.0"} ; Java implementation of the JQ json query languageorg.apache.commons/commons-compress {:mvn/version "1.28.0"} ; compression utilsorg.apache.commons/commons-lang3 {:mvn/version "3.20.0"} ; helper methods for working with java.lang stuff+ org.apache.datasketches/datasketches-java ^:antq/exclude ; Sketching algorithms (used for fingerprinting)+ {:mvn/version "7.0.1"} ; version is fixed to ensure JDK21 compatibility.org.apache.httpcomponents.client5/httpclient5 {:mvn/version "5.4.4"} ; pinned: ClickHouse JDBC 0.9.7 needs httpcore5 5.3.4+; must pair with httpclient5 5.4.xorg.apache.httpcomponents.core5/httpcore5 {:mvn/version "5.3.5"} ; pinned: ClickHouse JDBC 0.9.7 requires URIBuilder.optimize() added in 5.3org.apache.httpcomponents.core5/httpcore5-h2 {:mvn/version "5.3.5"} ; pinned: must match httpcore5 versiondiff --git a/src/metabase/analyze/fingerprint/fingerprinters.clj b/src/metabase/analyze/fingerprint/fingerprinters.cljindex 5f08cc765f9..4752e961bf0 100644--- a/src/metabase/analyze/fingerprint/fingerprinters.clj+++ b/src/metabase/analyze/fingerprint/fingerprinters.clj@@ -1,7 +1,6 @@(ns metabase.analyze.fingerprint.fingerprinters"Non-identifying fingerprinters for various field types."(:require- [bigml.histogram.core :as hist][clojure.string :as str][java-time.api :as t][kixi.stats.core :as stats]@@ -15,11 +14,13 @@[metabase.util.performance :as perf][redux.core :as redux])(:import- (com.bigml.histogram Histogram)- (com.clearspring.analytics.stream.cardinality HyperLogLogPlus)- (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZoneOffset ZonedDateTime)+ (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime ZoneOffset)(java.time.chrono ChronoLocalDateTime ChronoZonedDateTime)- (java.time.temporal ChronoField Temporal)))+ (java.time.temporal ChronoField Temporal)+ (org.apache.commons.codec.digest MurmurHash2)+ (org.apache.commons.math3.stat.descriptive SummaryStatistics)+ (org.apache.datasketches.hll HllSketch)+ (org.apache.datasketches.kll KllDoublesSketch)))(set! *warn-on-reflection* true)@@ -51,13 +52,17 @@([_ _] (reduced init))))(defn- cardinality- "Transducer that sketches cardinality using HyperLogLog++.- https://research.google.com/pubs/pub40671.html"- ([] (HyperLogLogPlus. 14 25))- ([^HyperLogLogPlus acc] (.cardinality acc))- ([^HyperLogLogPlus acc x]- (.offer acc x)- acc))+ "Transducer that sketches cardinality using DataSketches' HyperLogLog implementation."+ ([] (HllSketch. 16))+ ([^HllSketch acc] (Math/round (.getEstimate acc)))+ ([^HllSketch acc x]+ ;; Hashing is implemented in this way to ensure better overlap with results that the previously used+ ;; HyperLogLogPlus implementation produced.+ (let [h (cond (string? x) (MurmurHash2/hash64 ^String x)+ (bytes? x) (MurmurHash2/hash64 ^bytes x (alength ^bytes x))+ :else (hash x))]+ (.update acc ^long h)+ acc)))(defmacro robust-map"Wrap each map value in try-catch block."@@ -347,11 +352,17 @@(assoc :mode-fraction (:mode-fraction mode-stats):top-3-fraction (:top-3-fraction mode-stats)))))))-(defn- histogram- "Transducer that summarizes numerical data with a histogram."- ([] (hist/create))- ([^Histogram histogram] histogram)- ([^Histogram histogram x] (hist/insert-simple! histogram x)))+(deftype ^:private DistributionHolder [^SummaryStatistics summary, ^KllDoublesSketch kll])++(defn- distribution+ "Transducer that summarizes numerical data with SummaryStatistics and KllDoublesSketch."+ ([] (DistributionHolder. (SummaryStatistics.) (KllDoublesSketch/newHeapInstance)))+ ([^DistributionHolder holder] holder)+ ([^DistributionHolder holder x]+ (let [d (double x)]+ (.addValue ^SummaryStatistics (.summary holder) d)+ (.update ^KllDoublesSketch (.kll holder) d)+ holder)))(defprotocol ^:private INumberCoerceable"Protocol for converting objects to a java.lang.Number."@@ -370,16 +381,19 @@(deffingerprinter :type/Number(redux/post-complete((comp (map ->number) (filter u/real-number?))- (redux/juxt histogram stats/skewness stats/kurtosis mode-stats (stats/share zero?)))- (fn [[h sk ku ms zf]]- (let [{q1 0.25 q3 0.75} (hist/percentiles h 0.25 0.75)]+ (redux/juxt distribution stats/skewness stats/kurtosis mode-stats (stats/share zero?)))+ (fn [[^DistributionHolder h, sk ku ms zf]]+ (let [^SummaryStatistics summary (.summary h)+ ^KllDoublesSketch kll (.kll h)+ n (.getN summary)](robust-map- :min (hist/minimum h)- :max (hist/maximum h)- :avg (hist/mean h)- :sd (some-> h hist/variance math/sqrt)- :q1 q1- :q3 q3+ :min (.getMinItem kll)+ :max (.getMaxItem kll)+ ;; Ensure we don't get ##NaN in avg/sd+ :avg (when (pos? n) (.getMean summary))+ :sd (when (pos? n) (math/sqrt (.getVariance summary)))+ :q1 (.getQuantile kll 0.25)+ :q3 (.getQuantile kll 0.75):skewness sk:excess-kurtosis ku:mode-fraction (:mode-fraction ms)diff --git a/test/metabase/analyze/fingerprint/fingerprinters_test.clj b/test/metabase/analyze/fingerprint/fingerprinters_test.cljindex e93e9f21f98..a4eccc84611 100644--- a/test/metabase/analyze/fingerprint/fingerprinters_test.clj+++ b/test/metabase/analyze/fingerprint/fingerprinters_test.clj@@ -114,28 +114,28 @@[#t "2013" #t "2018" #t "2015"])))))))(deftest ^:parallel fingerprint-numeric-values-test- (is (= {:global {:distinct-count 3+ (is (= {:global {:distinct-count 99:nil% 0.0}- :type {:type/Number {:avg 2.0- :min 1.0- :max 3.0- :q1 1.25- :q3 2.75- :sd 1.0+ :type {:type/Number {:avg 49.0+ :min 0.0+ :max 98.0+ :q1 24.0+ :q3 74.0+ :sd 28.722813232690143:skewness 0.0- :excess-kurtosis nil- :mode-fraction (/ 1.0 3.0)- :top-3-fraction 1.0- :zero-fraction 0.0}}}+ :excess-kurtosis -1.2000000000000002+ :mode-fraction 0.010101010101010102+ :top-3-fraction 0.030303030303030304+ :zero-fraction 0.0101010101010101}}}(transduce identity(fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Number}))- [1.0 2.0 3.0])))+ (map double (range 99)))))(testing "we respect effective_type"(let [result (transduce identity(fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text :effective_type :type/Number}))["1" "2" "1.3" "2.3"])](is (= {:distinct-count 4, :nil% 0.0} (:global result)))- (is (= {:min 1.0 :q1 1.15 :q3 2.15 :max 2.3 :avg 1.65}+ (is (= {:min 1.0 :q1 1.0 :q3 2.0 :max 2.3 :avg 1.65}(select-keys (get-in result [:type :type/Number])[:min :q1 :q3 :max :avg])))(is (some? (get-in result [:type :type/Number :skewness])))@@ -145,9 +145,26 @@(fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Number}))[1.0 2.0 3.0 Double/NaN Double/POSITIVE_INFINITY Double/NEGATIVE_INFINITY nil nil])](is (= {:distinct-count 7, :nil% 0.25} (:global result)))- (is (= {:avg 2.0, :min 1.0, :max 3.0, :q1 1.25, :q3 2.75, :sd 1.0}+ (is (= {:avg 2.0, :min 1.0, :max 3.0, :q1 1.0, :q3 3.0, :sd 1.0}(select-keys (get-in result [:type :type/Number])- [:avg :min :max :q1 :q3 :sd]))))))+ [:avg :min :max :q1 :q3 :sd])))+ (is (=? {:global {:distinct-count 4, :nil% 0.0},+ :type {:type/Number {:min 1.0, :q1 1.0, :q3 2.0, :max 2.3, :sd 0.6027713773341707, :avg 1.65}}}+ (transduce identity+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text :effective_type :type/Number}))+ ["1" "2" "1.3" "2.3"])))))+ (testing "We should robustly survive weird values such as NaN, Infinity, and nil"+ (is (=? {:global {:distinct-count 7+ :nil% 0.25}+ :type {:type/Number {:avg 2.0+ :min 1.0+ :max 3.0+ :q1 1.0+ :q3 3.0+ :sd 1.0}}}+ (transduce identity+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Number}))+ [1.0 2.0 3.0 Double/NaN Double/POSITIVE_INFINITY Double/NEGATIVE_INFINITY nil nil])))))(deftest ^:parallel fingerprint-string-values-test(is (= {:global {:distinct-count 5@@ -214,11 +231,11 @@(is (=? {:global {:distinct-count 4:nil% 0.0}:type {:type/Number {:min 1.0- :q1 #(< 1.44 % 1.46)- :q3 #(< 2.4 % 2.5)+ :q1 2.0+ :q3 2.0:max 4.0:sd #(< 0.76 % 0.78)- :avg 2.03}}}+ :avg #(< 2.02 % 2.04)}}}(t2/select-one-fn :fingerprint :model/Field :id (mt/id :venues :price))))))))(deftest ^:parallel valid-serialized-json?-testdiff --git a/test/metabase/analyze/query_results_test.clj b/test/metabase/analyze/query_results_test.cljindex c2ee79e9389..6905cda7e6e 100644--- a/test/metabase/analyze/query_results_test.clj+++ b/test/metabase/analyze/query_results_test.clj@@ -85,7 +85,7 @@(mt/with-temp [:model/Card card {:dataset_query {:database (mt/id), :type :native, :native {:query "select * from venues"}}}](is (=? (assoc-in (mt/round-all-decimals 2 (app-db-venue-fingerprints))[:category_id :type]- #:type{:Number {:min 2.0, :max 74.0, :avg 29.98, :q1 6.9, :q3 49.24, :sd 23.06}})+ #:type{:Number {:min 2.0, :max 74.0, :avg 29.98, :q1 7.0, :q3 49.0, :sd 23.06}})(->> (name->fingerprints (query->result-metadata (query-for-card card)))(mt/round-all-decimals 2)))))))diff --git a/test/metabase/lib/test_metadata.cljc b/test/metabase/lib/test_metadata.cljcindex 860da57a9b1..1eac9c8e4ca 100644--- a/test/metabase/lib/test_metadata.cljc+++ b/test/metabase/lib/test_metadata.cljc@@ -808,8 +808,8 @@:fingerprint {:global {:distinct-count 4, :nil% 0.0}:type {:type/Number{:min 1.0- :q1 1.4591129021415095- :q3 2.493086095768049+ :q1 2.0+ :q3 2.0:max 4.0:sd 0.7713951678941896:avg 2.03}}}diff --git a/test/metabase/query_processor/middleware/results_metadata_test.clj b/test/metabase/query_processor/middleware/results_metadata_test.cljindex 39740e967de..a1e651fed4c 100644--- a/test/metabase/query_processor/middleware/results_metadata_test.clj+++ b/test/metabase/query_processor/middleware/results_metadata_test.clj@@ -90,8 +90,8 @@(update-in [3 :fingerprint] assoc :type {:type/Number {:min 2.0:max 74.0:avg 29.98- :q1 6.9- :q3 49.24+ :q1 7.0+ :q3 49.0:sd 23.06}}))](assoc column :display_name (:name column))))@@ -290,7 +290,7 @@:semantic_type :type/Quantity:fingerprint {:global {:distinct-count 3:nil% 0.0}- :type {:type/Number {:min 235.0, :max 498.0, :avg 333.33 :q1 243.0, :q3 440.25, :sd 143.5}}}+ :type {:type/Number {:min 235.0, :max 498.0, :avg 333.33 :q1 235.0, :q3 498.0, :sd 143.5}}}:field_ref [:aggregation 0]}](-> cardcard-metadata