Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Remove unused fingerprints (GHY-3636) (#74517)

Eric Normand019b86391ef2
diff --git a/src/metabase/analyze/fingerprint/fingerprinters.clj b/src/metabase/analyze/fingerprint/fingerprinters.clj
index 4752e961bf0..64e49ac7f2e 100644
--- a/src/metabase/analyze/fingerprint/fingerprinters.clj
+++ b/src/metabase/analyze/fingerprint/fingerprinters.clj
@@ -16,7 +16,7 @@
(:import
(java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime ZoneOffset)
(java.time.chrono ChronoLocalDateTime ChronoZonedDateTime)
- (java.time.temporal ChronoField Temporal)
+ (java.time.temporal Temporal)
(org.apache.commons.codec.digest MurmurHash2)
(org.apache.commons.math3.stat.descriptive SummaryStatistics)
(org.apache.datasketches.hll HllSketch)
@@ -283,69 +283,13 @@
(instance? OffsetTime t) (recur (t/offset-date-time (t/local-date "1970-01-01") t (t/zone-offset t)))
:else nil))
-(defn- temporal->zoned
- "Promote a `Temporal` to a form whose ChronoField support covers DAY_OF_WEEK and
- HOUR_OF_DAY. `Instant` is coerced to UTC; all other types pass through unchanged
- (e.g. `LocalDate` still supports DAY_OF_WEEK; `LocalTime` still supports HOUR_OF_DAY;
- downstream extractors check `isSupported` per-field)."
- ^Temporal [^Temporal t]
- (if (instance? Instant t)
- (.atZone ^Instant t ZoneOffset/UTC)
- t))
-
-(defn- temporal->weekday
- "Return 1-7 for Monday-Sunday, or nil if `t` is nil or the Temporal has no day-of-week."
- [t]
- (when-let [^Temporal z (some-> t temporal->zoned)]
- (when (.isSupported z ChronoField/DAY_OF_WEEK)
- (.get z ChronoField/DAY_OF_WEEK))))
-
-(defn- temporal->hour
- "Return 0-23 for hour-of-day, or nil if `t` is nil or the Temporal has no hour."
- [t]
- (when-let [^Temporal z (some-> t temporal->zoned)]
- (when (.isSupported z ChronoField/HOUR_OF_DAY)
- (.get z ChronoField/HOUR_OF_DAY))))
-
-(defn- bucket-distribution
- "Reducer factory: returns a reducer that bucketizes values via `bucket-fn` into `n-buckets`
- slots and, on completion, returns a vector of per-bucket fractions (summing to 1.0).
- Values where `bucket-fn` returns nil or out-of-range are skipped. Returns nil if no
- valid values were seen."
- [n-buckets bucket-fn]
- (fn
- ([] [(vec (repeat n-buckets 0)) 0])
- ([[buckets total]]
- (when (pos? total)
- (mapv #(/ (double %) (double total)) buckets)))
- ([[buckets total] x]
- (if-let [idx (bucket-fn x)]
- (if (and (<= 0 idx) (< idx n-buckets))
- [(update buckets idx inc) (inc total)]
- [buckets total])
- [buckets total]))))
-
-(defn- weekday-distribution
- "Reducer producing a 7-element vector of per-weekday fractions (Monday=index 0 .. Sunday=6)."
- []
- (bucket-distribution 7 (fn [t]
- (when-let [dow (temporal->weekday t)]
- (dec dow)))))
-
-(defn- hour-distribution
- "Reducer producing a 24-element vector of per-hour fractions (0..23)."
- []
- (bucket-distribution 24 temporal->hour))
-
(deffingerprinter :type/DateTime
((map ->temporal)
(redux/post-complete
- (robust-fuse {:earliest earliest
- :latest latest
- :skewness ((keep ->millis-from-epoch) stats/skewness)
- :mode-stats ((keep ->millis-from-epoch) mode-stats)
- :weekday-distribution (weekday-distribution)
- :hour-distribution (hour-distribution)})
+ (robust-fuse {:earliest earliest
+ :latest latest
+ :skewness ((keep ->millis-from-epoch) stats/skewness)
+ :mode-stats ((keep ->millis-from-epoch) mode-stats)})
(fn [{:keys [mode-stats] :as fused}]
(-> fused
(dissoc :mode-stats)
@@ -427,8 +371,6 @@
:percent-state (stats/share u/state?)
:percent-blank (stats/share str/blank?)
:average-length ((map count) stats/mean)
- :min-length ((map count) stats/min)
- :max-length ((map count) stats/max)
:mode-stats mode-stats})
(fn [{:keys [mode-stats] :as fused}]
(-> fused
diff --git a/src/metabase/lib/schema/metadata/fingerprint.cljc b/src/metabase/lib/schema/metadata/fingerprint.cljc
index 63b7c6ca851..ab7f3a0b303 100644
--- a/src/metabase/lib/schema/metadata/fingerprint.cljc
+++ b/src/metabase/lib/schema/metadata/fingerprint.cljc
@@ -44,8 +44,6 @@
[:percent-state {:optional true} [:maybe [:ref ::percent]]]
[:percent-blank {:optional true} [:maybe [:ref ::percent]]]
[:average-length {:optional true} [:maybe number?]]
- [:min-length {:optional true} [:maybe number?]]
- [:max-length {:optional true} [:maybe number?]]
[:mode-fraction {:optional true} [:maybe [:ref ::percent]]]
[:top-3-fraction {:optional true} [:maybe [:ref ::percent]]]])
@@ -57,9 +55,7 @@
[:latest {:optional true} [:maybe :string]]
[:skewness {:optional true} [:maybe number?]]
[:mode-fraction {:optional true} [:maybe [:ref ::percent]]]
- [:top-3-fraction {:optional true} [:maybe [:ref ::percent]]]
- [:weekday-distribution {:optional true} [:maybe [:sequential [:ref ::percent]]]]
- [:hour-distribution {:optional true} [:maybe [:sequential [:ref ::percent]]]]])
+ [:top-3-fraction {:optional true} [:maybe [:ref ::percent]]]])
(mr/def ::fingerprint.type-specific
"Schema for type-specific fingerprint information."
diff --git a/test/metabase/analyze/fingerprint/fingerprinters_test.clj b/test/metabase/analyze/fingerprint/fingerprinters_test.clj
index a4eccc84611..b09b9659e8f 100644
--- a/test/metabase/analyze/fingerprint/fingerprinters_test.clj
+++ b/test/metabase/analyze/fingerprint/fingerprinters_test.clj
@@ -15,7 +15,7 @@
(defn- temporal-core-fields
"Return the subset of a temporal fingerprint result containing the historically-asserted fields.
We compare only :earliest / :latest (plus :global) to keep tests stable against new distribution
- stats (skewness, mode-fraction, weekday-distribution, hour-distribution)."
+ stats (skewness, mode-fraction)."
[result]
{:global (:global result)
:type {:type/DateTime (select-keys (get-in result [:type :type/DateTime])
@@ -174,8 +174,6 @@
:percent-email 0.0
:percent-state 0.0
:average-length 6.4
- :min-length 4.0
- :max-length 9.0
:mode-fraction 0.2
:top-3-fraction 0.6
:percent-blank 0.0}}}
@@ -190,8 +188,6 @@
:percent-email 0.0
:percent-state 0.0
:average-length 10.6
- :min-length 4.0
- :max-length 30.0
:mode-fraction 0.2
:top-3-fraction 0.6
:percent-blank 0.0}}}
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index c9a130c3a3f..5b1cc017a74 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -1404,10 +1404,7 @@
:table_id (t2/select-one-pk :model/Table :db_id (u/the-id database)))
;; Strip extended interestingness stats — this test covers the core TIME fingerprint
;; shape (#5911), not the interestingness metrics.
- extended-keys [:hour-distribution :weekday-distribution :skewness
- :mode-fraction :top-3-fraction
- :mode-fraction-by-weekday :mode-fraction-by-hour
- :min-length :max-length :percent-blank]
+ extended-keys [:skewness :mode-fraction :top-3-fraction :percent-blank]
trim-type (fn [fp]
(update fp :type
(fn [types]
diff --git a/test/metabase/metabot/tools/field_stats_test.clj b/test/metabase/metabot/tools/field_stats_test.clj
index c7000298cec..b73b2b69096 100644
--- a/test/metabase/metabot/tools/field_stats_test.clj
+++ b/test/metabase/metabot/tools/field_stats_test.clj
@@ -49,14 +49,12 @@
(get-in result [:structured-output :value_metadata :statistics :skewness])
(update-in [:structured-output :value_metadata :statistics] dissoc :skewness))))
people-id birth-date-id {:statistics
- {:distinct-count 2308
- :percent-null 0.0
- :earliest "1958-04-26"
- :latest "2000-04-03"
- :hour-distribution nil
- :mode-fraction 0.0012
- :top-3-fraction 0.0032
- :weekday-distribution [0.15 0.1304 0.1416 0.1372 0.156 0.1516 0.1332]}}
+ {:distinct-count 2308
+ :percent-null 0.0
+ :earliest "1958-04-26"
+ :latest "2000-04-03"
+ :mode-fraction 0.0012
+ :top-3-fraction 0.0032}}
people-id state-id {:statistics {:distinct-count 49
:percent-null 0.0
:percent-json 0.0
@@ -64,8 +62,6 @@
:percent-email 0.0
:percent-state 1.0
:average-length 2.0
- :max-length 2.0
- :min-length 2.0
:mode-fraction 0.0776
:top-3-fraction 0.1624
:percent-blank 0.0}
@@ -77,8 +73,6 @@
:percent-email 0.0
:percent-state 0.0
:average-length 6.375
- :max-length 9.0
- :min-length 5.0
:mode-fraction 0.27
:top-3-fraction 0.79
:percent-blank 0.0}
diff --git a/test/metabase/sample_data/impl_test.clj b/test/metabase/sample_data/impl_test.clj
index 44b5ccf36b7..8dda55d2eda 100644
--- a/test/metabase/sample_data/impl_test.clj
+++ b/test/metabase/sample_data/impl_test.clj
@@ -98,8 +98,6 @@
:percent-email 0.0
:percent-state 0.0
:average-length 13.532
- :max-length 23.0
- :min-length 7.0
:mode-fraction 4.0E-4
:top-3-fraction 0.0012
:percent-blank 0.0}}}