The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Remove unused fingerprints (GHY-3636) (#74517)
Eric Normand019b86391ef2
diff --git a/src/metabase/analyze/fingerprint/fingerprinters.clj b/src/metabase/analyze/fingerprint/fingerprinters.cljindex 4752e961bf0..64e49ac7f2e 100644--- a/src/metabase/analyze/fingerprint/fingerprinters.clj+++ b/src/metabase/analyze/fingerprint/fingerprinters.clj@@ -16,7 +16,7 @@(:import(java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime ZoneOffset)(java.time.chrono ChronoLocalDateTime ChronoZonedDateTime)- (java.time.temporal ChronoField Temporal)+ (java.time.temporal Temporal)(org.apache.commons.codec.digest MurmurHash2)(org.apache.commons.math3.stat.descriptive SummaryStatistics)(org.apache.datasketches.hll HllSketch)@@ -283,69 +283,13 @@(instance? OffsetTime t) (recur (t/offset-date-time (t/local-date "1970-01-01") t (t/zone-offset t))):else nil))-(defn- temporal->zoned- "Promote a `Temporal` to a form whose ChronoField support covers DAY_OF_WEEK and- HOUR_OF_DAY. `Instant` is coerced to UTC; all other types pass through unchanged- (e.g. `LocalDate` still supports DAY_OF_WEEK; `LocalTime` still supports HOUR_OF_DAY;- downstream extractors check `isSupported` per-field)."- ^Temporal [^Temporal t]- (if (instance? Instant t)- (.atZone ^Instant t ZoneOffset/UTC)- t))--(defn- temporal->weekday- "Return 1-7 for Monday-Sunday, or nil if `t` is nil or the Temporal has no day-of-week."- [t]- (when-let [^Temporal z (some-> t temporal->zoned)]- (when (.isSupported z ChronoField/DAY_OF_WEEK)- (.get z ChronoField/DAY_OF_WEEK))))--(defn- temporal->hour- "Return 0-23 for hour-of-day, or nil if `t` is nil or the Temporal has no hour."- [t]- (when-let [^Temporal z (some-> t temporal->zoned)]- (when (.isSupported z ChronoField/HOUR_OF_DAY)- (.get z ChronoField/HOUR_OF_DAY))))--(defn- bucket-distribution- "Reducer factory: returns a reducer that bucketizes values via `bucket-fn` into `n-buckets`- slots and, on completion, returns a vector of per-bucket fractions (summing to 1.0).- Values where `bucket-fn` returns nil or out-of-range are skipped. Returns nil if no- valid values were seen."- [n-buckets bucket-fn]- (fn- ([] [(vec (repeat n-buckets 0)) 0])- ([[buckets total]]- (when (pos? total)- (mapv #(/ (double %) (double total)) buckets)))- ([[buckets total] x]- (if-let [idx (bucket-fn x)]- (if (and (<= 0 idx) (< idx n-buckets))- [(update buckets idx inc) (inc total)]- [buckets total])- [buckets total]))))--(defn- weekday-distribution- "Reducer producing a 7-element vector of per-weekday fractions (Monday=index 0 .. Sunday=6)."- []- (bucket-distribution 7 (fn [t]- (when-let [dow (temporal->weekday t)]- (dec dow)))))--(defn- hour-distribution- "Reducer producing a 24-element vector of per-hour fractions (0..23)."- []- (bucket-distribution 24 temporal->hour))-(deffingerprinter :type/DateTime((map ->temporal)(redux/post-complete- (robust-fuse {:earliest earliest- :latest latest- :skewness ((keep ->millis-from-epoch) stats/skewness)- :mode-stats ((keep ->millis-from-epoch) mode-stats)- :weekday-distribution (weekday-distribution)- :hour-distribution (hour-distribution)})+ (robust-fuse {:earliest earliest+ :latest latest+ :skewness ((keep ->millis-from-epoch) stats/skewness)+ :mode-stats ((keep ->millis-from-epoch) mode-stats)})(fn [{:keys [mode-stats] :as fused}](-> fused(dissoc :mode-stats)@@ -427,8 +371,6 @@:percent-state (stats/share u/state?):percent-blank (stats/share str/blank?):average-length ((map count) stats/mean)- :min-length ((map count) stats/min)- :max-length ((map count) stats/max):mode-stats mode-stats})(fn [{:keys [mode-stats] :as fused}](-> fuseddiff --git a/src/metabase/lib/schema/metadata/fingerprint.cljc b/src/metabase/lib/schema/metadata/fingerprint.cljcindex 63b7c6ca851..ab7f3a0b303 100644--- a/src/metabase/lib/schema/metadata/fingerprint.cljc+++ b/src/metabase/lib/schema/metadata/fingerprint.cljc@@ -44,8 +44,6 @@[:percent-state {:optional true} [:maybe [:ref ::percent]]][:percent-blank {:optional true} [:maybe [:ref ::percent]]][:average-length {:optional true} [:maybe number?]]- [:min-length {:optional true} [:maybe number?]]- [:max-length {:optional true} [:maybe number?]][:mode-fraction {:optional true} [:maybe [:ref ::percent]]][:top-3-fraction {:optional true} [:maybe [:ref ::percent]]]])@@ -57,9 +55,7 @@[:latest {:optional true} [:maybe :string]][:skewness {:optional true} [:maybe number?]][:mode-fraction {:optional true} [:maybe [:ref ::percent]]]- [:top-3-fraction {:optional true} [:maybe [:ref ::percent]]]- [:weekday-distribution {:optional true} [:maybe [:sequential [:ref ::percent]]]]- [:hour-distribution {:optional true} [:maybe [:sequential [:ref ::percent]]]]])+ [:top-3-fraction {:optional true} [:maybe [:ref ::percent]]]])(mr/def ::fingerprint.type-specific"Schema for type-specific fingerprint information."diff --git a/test/metabase/analyze/fingerprint/fingerprinters_test.clj b/test/metabase/analyze/fingerprint/fingerprinters_test.cljindex a4eccc84611..b09b9659e8f 100644--- a/test/metabase/analyze/fingerprint/fingerprinters_test.clj+++ b/test/metabase/analyze/fingerprint/fingerprinters_test.clj@@ -15,7 +15,7 @@(defn- temporal-core-fields"Return the subset of a temporal fingerprint result containing the historically-asserted fields.We compare only :earliest / :latest (plus :global) to keep tests stable against new distribution- stats (skewness, mode-fraction, weekday-distribution, hour-distribution)."+ stats (skewness, mode-fraction)."[result]{:global (:global result):type {:type/DateTime (select-keys (get-in result [:type :type/DateTime])@@ -174,8 +174,6 @@:percent-email 0.0:percent-state 0.0:average-length 6.4- :min-length 4.0- :max-length 9.0:mode-fraction 0.2:top-3-fraction 0.6:percent-blank 0.0}}}@@ -190,8 +188,6 @@:percent-email 0.0:percent-state 0.0:average-length 10.6- :min-length 4.0- :max-length 30.0:mode-fraction 0.2:top-3-fraction 0.6:percent-blank 0.0}}}diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex c9a130c3a3f..5b1cc017a74 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -1404,10 +1404,7 @@:table_id (t2/select-one-pk :model/Table :db_id (u/the-id database)));; Strip extended interestingness stats — this test covers the core TIME fingerprint;; shape (#5911), not the interestingness metrics.- extended-keys [:hour-distribution :weekday-distribution :skewness- :mode-fraction :top-3-fraction- :mode-fraction-by-weekday :mode-fraction-by-hour- :min-length :max-length :percent-blank]+ extended-keys [:skewness :mode-fraction :top-3-fraction :percent-blank]trim-type (fn [fp](update fp :type(fn [types]diff --git a/test/metabase/metabot/tools/field_stats_test.clj b/test/metabase/metabot/tools/field_stats_test.cljindex c7000298cec..b73b2b69096 100644--- a/test/metabase/metabot/tools/field_stats_test.clj+++ b/test/metabase/metabot/tools/field_stats_test.clj@@ -49,14 +49,12 @@(get-in result [:structured-output :value_metadata :statistics :skewness])(update-in [:structured-output :value_metadata :statistics] dissoc :skewness))))people-id birth-date-id {:statistics- {:distinct-count 2308- :percent-null 0.0- :earliest "1958-04-26"- :latest "2000-04-03"- :hour-distribution nil- :mode-fraction 0.0012- :top-3-fraction 0.0032- :weekday-distribution [0.15 0.1304 0.1416 0.1372 0.156 0.1516 0.1332]}}+ {:distinct-count 2308+ :percent-null 0.0+ :earliest "1958-04-26"+ :latest "2000-04-03"+ :mode-fraction 0.0012+ :top-3-fraction 0.0032}}people-id state-id {:statistics {:distinct-count 49:percent-null 0.0:percent-json 0.0@@ -64,8 +62,6 @@:percent-email 0.0:percent-state 1.0:average-length 2.0- :max-length 2.0- :min-length 2.0:mode-fraction 0.0776:top-3-fraction 0.1624:percent-blank 0.0}@@ -77,8 +73,6 @@:percent-email 0.0:percent-state 0.0:average-length 6.375- :max-length 9.0- :min-length 5.0:mode-fraction 0.27:top-3-fraction 0.79:percent-blank 0.0}diff --git a/test/metabase/sample_data/impl_test.clj b/test/metabase/sample_data/impl_test.cljindex 44b5ccf36b7..8dda55d2eda 100644--- a/test/metabase/sample_data/impl_test.clj+++ b/test/metabase/sample_data/impl_test.clj@@ -98,8 +98,6 @@:percent-email 0.0:percent-state 0.0:average-length 13.532- :max-length 23.0- :min-length 7.0:mode-fraction 4.0E-4:top-3-fraction 0.0012:percent-blank 0.0}}}