Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Dimension and chart interestiningness calculation functions (#72630)

Edward Pagetcbeb2d4be61a
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 2115d88c8b7..8e9cdf7cb12 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1020,6 +1020,11 @@
:api #{metabase.initialization-status.core}
:uses #{}}
+ interestingness
+ {:team "UX West"
+ :api #{metabase.interestingness.core}
+ :uses #{util}}
+
internal-stats
{:team "UX West"
:api #{metabase.internal-stats.core}
@@ -1223,6 +1228,7 @@
config
documents
driver
+ interestingness
legacy-mbql
lib
lib-be
@@ -2119,6 +2125,7 @@
database-routing
driver
events
+ interestingness
lib
models
premium-features
diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.clj
index ba833f95794..2c84837010e 100644
--- a/mage/src/mage/modules.clj
+++ b/mage/src/mage/modules.clj
@@ -116,6 +116,7 @@
events
formatter
initialization-status
+ interestingness
internal-stats
llm
login-history
diff --git a/resources/migrations/061/20260417_interestingness.yaml b/resources/migrations/061/20260417_interestingness.yaml
new file mode 100644
index 00000000000..c3620ae5fe2
--- /dev/null
+++ b/resources/migrations/061/20260417_interestingness.yaml
@@ -0,0 +1,16 @@
+databaseChangeLog:
+ - changeSet:
+ id: v61.int3r2
+ author: nvoxland
+ comment: Add dimension interestingness score to metabase_field
+ changes:
+ - addColumn:
+ tableName: metabase_field
+ columns:
+ - column:
+ name: dimension_interestingness
+ type: double precision
+ remarks: >-
+ Interestingness score (0.0-1.0) for this field when treated as a dimension in a metric.
+ constraints:
+ nullable: true
diff --git a/src/metabase/analyze/fingerprint/fingerprinters.clj b/src/metabase/analyze/fingerprint/fingerprinters.clj
index 8fe350f2725..5f08cc765f9 100644
--- a/src/metabase/analyze/fingerprint/fingerprinters.clj
+++ b/src/metabase/analyze/fingerprint/fingerprinters.clj
@@ -2,6 +2,7 @@
"Non-identifying fingerprinters for various field types."
(:require
[bigml.histogram.core :as hist]
+ [clojure.string :as str]
[java-time.api :as t]
[kixi.stats.core :as stats]
[kixi.stats.math :as math]
@@ -16,9 +17,9 @@
(:import
(com.bigml.histogram Histogram)
(com.clearspring.analytics.stream.cardinality HyperLogLogPlus)
- (java.time ZoneOffset)
+ (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZoneOffset ZonedDateTime)
(java.time.chrono ChronoLocalDateTime ChronoZonedDateTime)
- (java.time.temporal Temporal)))
+ (java.time.temporal ChronoField Temporal)))
(set! *warn-on-reflection* true)
@@ -232,10 +233,119 @@
Temporal (->temporal [this] this)
java.util.Date (->temporal [this] (t/instant this)))
+(def ^:private mode-stats-max-distinct
+ "Maximum number of distinct values to track for mode-fraction / top-3-fraction computation.
+ Values beyond this cap are counted toward total but not tracked individually. For typical
+ 10k-row samples, the true top values will almost always appear in the first 100 distinct
+ values seen."
+ 100)
+
+(defn- top-n-fraction
+ "Given a map of value->count and a total row count, return the sum of the top-n counts
+ divided by total. Returns nil if total is not positive."
+ [counts total n]
+ (when (pos? total)
+ (/ (double (reduce + 0.0 (take n (sort > (vals counts)))))
+ (double total))))
+
+(defn- mode-stats
+ "Reducer that tracks value frequencies (bounded at `mode-stats-max-distinct` entries)
+ and, on completion, returns {:mode-fraction, :top-3-fraction}. High mode-fraction
+ signals single-value dominance; high top-3-fraction with moderate mode-fraction signals
+ bimodal / few-real-categories distributions."
+ ([] [{} 0])
+ ([[counts total]]
+ {:mode-fraction (top-n-fraction counts total 1)
+ :top-3-fraction (top-n-fraction counts total 3)})
+ ([[counts total] x]
+ (cond
+ (nil? x) [counts (inc total)]
+ (contains? counts x) [(update counts x inc) (inc total)]
+ (< (count counts) mode-stats-max-distinct) [(assoc counts x 1) (inc total)]
+ :else [counts (inc total)])))
+
+(defn- ->millis-from-epoch
+ "Coerce a `java.time.temporal.Temporal` (as produced by `->temporal`) to long epoch millis.
+ Returns nil for unsupported types; callers typically wrap with `(keep ->millis-from-epoch)`
+ to strip non-coerceable values."
+ [t]
+ (cond (instance? Instant t) (.toEpochMilli ^Instant t)
+ (instance? OffsetDateTime t) (.toEpochMilli (.toInstant ^OffsetDateTime t))
+ (instance? ZonedDateTime t) (.toEpochMilli (.toInstant ^ZonedDateTime t))
+ (instance? LocalDate t) (recur (t/offset-date-time t (t/local-time 0) (t/zone-offset 0)))
+ (instance? LocalDateTime t) (recur (t/offset-date-time t (t/zone-offset 0)))
+ (instance? LocalTime t) (recur (t/offset-date-time (t/local-date "1970-01-01") t (t/zone-offset 0)))
+ (instance? OffsetTime t) (recur (t/offset-date-time (t/local-date "1970-01-01") t (t/zone-offset t)))
+ :else nil))
+
+(defn- temporal->zoned
+ "Promote a `Temporal` to a form whose ChronoField support covers DAY_OF_WEEK and
+ HOUR_OF_DAY. `Instant` is coerced to UTC; all other types pass through unchanged
+ (e.g. `LocalDate` still supports DAY_OF_WEEK; `LocalTime` still supports HOUR_OF_DAY;
+ downstream extractors check `isSupported` per-field)."
+ ^Temporal [^Temporal t]
+ (if (instance? Instant t)
+ (.atZone ^Instant t ZoneOffset/UTC)
+ t))
+
+(defn- temporal->weekday
+ "Return 1-7 for Monday-Sunday, or nil if `t` is nil or the Temporal has no day-of-week."
+ [t]
+ (when-let [^Temporal z (some-> t temporal->zoned)]
+ (when (.isSupported z ChronoField/DAY_OF_WEEK)
+ (.get z ChronoField/DAY_OF_WEEK))))
+
+(defn- temporal->hour
+ "Return 0-23 for hour-of-day, or nil if `t` is nil or the Temporal has no hour."
+ [t]
+ (when-let [^Temporal z (some-> t temporal->zoned)]
+ (when (.isSupported z ChronoField/HOUR_OF_DAY)
+ (.get z ChronoField/HOUR_OF_DAY))))
+
+(defn- bucket-distribution
+ "Reducer factory: returns a reducer that bucketizes values via `bucket-fn` into `n-buckets`
+ slots and, on completion, returns a vector of per-bucket fractions (summing to 1.0).
+ Values where `bucket-fn` returns nil or out-of-range are skipped. Returns nil if no
+ valid values were seen."
+ [n-buckets bucket-fn]
+ (fn
+ ([] [(vec (repeat n-buckets 0)) 0])
+ ([[buckets total]]
+ (when (pos? total)
+ (mapv #(/ (double %) (double total)) buckets)))
+ ([[buckets total] x]
+ (if-let [idx (bucket-fn x)]
+ (if (and (<= 0 idx) (< idx n-buckets))
+ [(update buckets idx inc) (inc total)]
+ [buckets total])
+ [buckets total]))))
+
+(defn- weekday-distribution
+ "Reducer producing a 7-element vector of per-weekday fractions (Monday=index 0 .. Sunday=6)."
+ []
+ (bucket-distribution 7 (fn [t]
+ (when-let [dow (temporal->weekday t)]
+ (dec dow)))))
+
+(defn- hour-distribution
+ "Reducer producing a 24-element vector of per-hour fractions (0..23)."
+ []
+ (bucket-distribution 24 temporal->hour))
+
(deffingerprinter :type/DateTime
((map ->temporal)
- (robust-fuse {:earliest earliest
- :latest latest})))
+ (redux/post-complete
+ (robust-fuse {:earliest earliest
+ :latest latest
+ :skewness ((keep ->millis-from-epoch) stats/skewness)
+ :mode-stats ((keep ->millis-from-epoch) mode-stats)
+ :weekday-distribution (weekday-distribution)
+ :hour-distribution (hour-distribution)})
+ (fn [{:keys [mode-stats] :as fused}]
+ (-> fused
+ (dissoc :mode-stats)
+ (assoc :mode-fraction (:mode-fraction mode-stats)
+ :top-3-fraction (:top-3-fraction mode-stats)))))))
(defn- histogram
"Transducer that summarizes numerical data with a histogram."
@@ -259,16 +369,22 @@
(deffingerprinter :type/Number
(redux/post-complete
- ((comp (map ->number) (filter u/real-number?)) histogram)
- (fn [h]
+ ((comp (map ->number) (filter u/real-number?))
+ (redux/juxt histogram stats/skewness stats/kurtosis mode-stats (stats/share zero?)))
+ (fn [[h sk ku ms zf]]
(let [{q1 0.25 q3 0.75} (hist/percentiles h 0.25 0.75)]
(robust-map
- :min (hist/minimum h)
- :max (hist/maximum h)
- :avg (hist/mean h)
- :sd (some-> h hist/variance math/sqrt)
- :q1 q1
- :q3 q3)))))
+ :min (hist/minimum h)
+ :max (hist/maximum h)
+ :avg (hist/mean h)
+ :sd (some-> h hist/variance math/sqrt)
+ :q1 q1
+ :q3 q3
+ :skewness sk
+ :excess-kurtosis ku
+ :mode-fraction (:mode-fraction ms)
+ :top-3-fraction (:top-3-fraction ms)
+ :zero-fraction zf)))))
(defn- valid-serialized-json?
"Is x a serialized JSON dictionary or array. Hueristically recognize maps and arrays. Uses the following strategies:
@@ -290,11 +406,21 @@
((map str) ; we cast to str to support `field-literal` type overwriting:
; `[:field-literal "A_NUMBER" :type/Text]` (which still
; returns numbers in the result set)
- (robust-fuse {:percent-json (stats/share valid-serialized-json?)
- :percent-url (stats/share u/url?)
- :percent-email (stats/share u/email?)
- :percent-state (stats/share u/state?)
- :average-length ((map count) stats/mean)})))
+ (redux/post-complete
+ (robust-fuse {:percent-json (stats/share valid-serialized-json?)
+ :percent-url (stats/share u/url?)
+ :percent-email (stats/share u/email?)
+ :percent-state (stats/share u/state?)
+ :percent-blank (stats/share str/blank?)
+ :average-length ((map count) stats/mean)
+ :min-length ((map count) stats/min)
+ :max-length ((map count) stats/max)
+ :mode-stats mode-stats})
+ (fn [{:keys [mode-stats] :as fused}]
+ (-> fused
+ (dissoc :mode-stats)
+ (assoc :mode-fraction (:mode-fraction mode-stats)
+ :top-3-fraction (:top-3-fraction mode-stats)))))))
(defn fingerprint-fields
"Return a transducer for fingerprinting a resultset with fields `fields`."
diff --git a/src/metabase/interestingness/chart.clj b/src/metabase/interestingness/chart.clj
new file mode 100644
index 00000000000..f74869f4792
--- /dev/null
+++ b/src/metabase/interestingness/chart.clj
@@ -0,0 +1,169 @@
+(ns metabase.interestingness.chart
+ "Chart-level interestingness scoring.
+
+ Operates on the same `chart-config` shape that
+ [[metabase.interestingness.chart.stats/compute-chart-stats]] consumes —
+ `{:series {name -> {:x :y :x_values :y_values ...}} :display_type ...}` —
+ and returns a scalar score in `[0.0, 1.0]` describing how likely the chart
+ is to surface a useful insight.
+
+ The score blends three signal groups:
+ - **non-degeneracy**: hard gate against flat measures and single-value
+ dimensions (a chart no aggregation can rescue)
+ - **statistical signal**: presence of trend, outliers, correlations, peaks
+ in the distribution — read from the stats map
+ - **structure**: reasonable category count, non-trivial data-point count
+
+ Callers who already compute stats can pass them via the 2-arity form to
+ avoid recomputation."
+ (:require
+ [metabase.interestingness.chart.stats :as chart.stats]))
+
+(set! *warn-on-reflection* true)
+
+(defn- variance
+ [xs]
+ (let [n (count xs)]
+ (when (pos? n)
+ (let [mean (/ (double (reduce + 0.0 xs)) n)]
+ (/ (reduce + 0.0 (map #(let [d (- % mean)] (* d d)) xs)) n)))))
+
+(defn- flat?
+ "True if the series' y_values have effectively zero spread."
+ [{:keys [y_values], :as _series}]
+ (let [v (variance y_values)]
+ (or (nil? v) (< v 1e-12))))
+
+(defn- single-valued-x?
+ [{:keys [x_values]}]
+ (<= (count (distinct x_values)) 1))
+
+(defn- non-degeneracy-score
+ "0 if every series is flat or has a single x; softer penalty if any one is.
+ 1 when all series look lively."
+ [{:keys [series], :as _chart-config}]
+ (if (empty? series)
+ 0.0
+ (let [configs (vals series)
+ all-flat? (every? flat? configs)
+ any-flat? (some flat? configs)
+ all-single? (every? single-valued-x? configs)
+ any-single? (some single-valued-x? configs)]
+ (cond
+ (or all-flat? all-single?) 0.0
+ (or any-flat? any-single?) 0.4
+ :else 1.0))))
+
+(defmulti ^:private signal-score
+ "Read statistical signals from a computed stats map. Returns 0..1."
+ {:arglists '([stats])}
+ :chart-type)
+
+(defn- trend-interesting?
+ [{:keys [direction overall-change-pct]}]
+ (or (contains? #{:strongly-increasing :strongly-decreasing} direction)
+ (and (contains? #{:increasing :decreasing} direction)
+ (some-> overall-change-pct double abs (>= 5.0)))))
+
+(defmethod signal-score :time-series
+ [{:keys [series correlations], :as _stats}]
+ (if (empty? series)
+ 0.0
+ (let [per-series (for [[_ s] series]
+ (let [has-trend (some-> (:trend s) trend-interesting?)
+ has-outlier (seq (:outliers s))
+ has-change (seq (:significant-changes s))
+ volatile? (contains? #{:high :extreme}
+ (some-> (:volatility s) :level))]
+ (cond-> 0.0
+ has-trend (+ 0.4)
+ has-outlier (+ 0.25)
+ has-change (+ 0.2)
+ volatile? (+ 0.15))))
+ base (/ (reduce + 0.0 per-series) (count per-series))
+ corr-bonus (if (some #(= :strong (:strength %)) correlations) 0.15 0.0)]
+ (min 1.0 (+ base corr-bonus)))))
+
+(defmethod signal-score :categorical
+ [{:keys [series]}]
+ (if (empty? series)
+ 0.0
+ (let [scores (for [[_ s] series]
+ (let [cc (:category-count s 0)
+ has-outlier (seq (:outliers s))
+ range-ratio (when-let [{:keys [min max]} (:summary s)]
+ (when (and min max (pos? (Math/abs (double max))))
+ (/ (- (double max) (double min))
+ (Math/abs (double max)))))]
+ (cond-> 0.0
+ (<= 3 cc 50) (+ 0.4)
+ (< 50 cc 200) (+ 0.2)
+ has-outlier (+ 0.2)
+ (and range-ratio (> range-ratio 0.25)) (+ 0.25))))]
+ (min 1.0 (/ (reduce + 0.0 scores) (count scores))))))
+
+(defmethod signal-score :scatter
+ [{:keys [series]}]
+ (if (empty? series)
+ 0.0
+ (let [scores (for [[_ s] series]
+ (let [coef (some-> s :correlation :coefficient double abs)
+ r2 (some-> s :regression :r-squared double)
+ has-outlier (seq (:outliers s))]
+ (cond-> 0.0
+ (and coef (>= coef 0.7)) (+ 0.5)
+ (and coef (< 0.4 coef 0.7)) (+ 0.3)
+ (and r2 (>= r2 0.3)) (+ 0.2)
+ has-outlier (+ 0.15))))]
+ (min 1.0 (/ (reduce + 0.0 scores) (count scores))))))
+
+(defmethod signal-score :histogram
+ [{:keys [series]}]
+ (if (empty? series)
+ 0.0
+ (let [scores (for [[_ s] series]
+ (let [{:keys [peak-count concentration-top3]} (:structure s)
+ skew (some-> s :distribution :weighted-skewness double abs)]
+ (cond-> 0.0
+ (and peak-count (>= peak-count 2)) (+ 0.35)
+ (and concentration-top3 (> concentration-top3 0.5)) (+ 0.25)
+ (and skew (> skew 1.0)) (+ 0.3))))]
+ (min 1.0 (/ (reduce + 0.0 scores) (count scores))))))
+
+(defmethod signal-score :default
+ [_]
+ 0.2)
+
+(defn- structure-score
+ "Reward reasonable data shape (enough points, not too many empty series)."
+ [chart-config]
+ (let [configs (vals (:series chart-config))
+ point-counts (map #(count (:y_values %)) configs)
+ total (reduce + 0 point-counts)]
+ (cond
+ (empty? configs) 0.0
+ (< total 3) 0.1
+ (< total 10) 0.5
+ (< total 10000) 1.0
+ :else 0.8)))
+
+(defn chart-interestingness
+ "Compute a `[0.0, 1.0]` interestingness score for a chart.
+
+ `chart-config` is the same map shape consumed by
+ [[metabase.interestingness.chart.stats/compute-chart-stats]]. Optional
+ `stats` lets callers who already computed stats avoid recomputation."
+ ([chart-config]
+ (chart-interestingness chart-config
+ (chart.stats/compute-chart-stats chart-config {})))
+ ([chart-config stats]
+ (let [nd (non-degeneracy-score chart-config)]
+ (if (zero? nd)
+ 0.0
+ (let [sig (signal-score stats)
+ struct (structure-score chart-config)]
+ (min 1.0
+ (max 0.0
+ (+ (* 0.3 nd)
+ (* 0.5 sig)
+ (* 0.2 struct)))))))))
diff --git a/src/metabase/metabot/stats/categorical.clj b/src/metabase/interestingness/chart/categorical.clj
similarity index 92%
rename from src/metabase/metabot/stats/categorical.clj
rename to src/metabase/interestingness/chart/categorical.clj
index 1167e21863a..97f426764e4 100644
--- a/src/metabase/metabot/stats/categorical.clj
+++ b/src/metabase/interestingness/chart/categorical.clj
@@ -1,9 +1,9 @@
-(ns metabase.metabot.stats.categorical
+(ns metabase.interestingness.chart.categorical
"Categorical chart statistics."
(:require
- [metabase.metabot.stats.outliers :as outliers]
- [metabase.metabot.stats.types :as stats.types]
- [metabase.metabot.stats.util :as stats.u]
+ [metabase.interestingness.chart.outliers :as outliers]
+ [metabase.interestingness.chart.types :as stats.types]
+ [metabase.interestingness.chart.util :as stats.u]
[metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
diff --git a/src/metabase/metabot/stats/histogram.clj b/src/metabase/interestingness/chart/histogram.clj
similarity index 98%
rename from src/metabase/metabot/stats/histogram.clj
rename to src/metabase/interestingness/chart/histogram.clj
index 6b1e42ceb2f..888251fec8b 100644
--- a/src/metabase/metabot/stats/histogram.clj
+++ b/src/metabase/interestingness/chart/histogram.clj
@@ -1,8 +1,8 @@
-(ns metabase.metabot.stats.histogram
+(ns metabase.interestingness.chart.histogram
"Histogram chart statistics."
(:require
- [metabase.metabot.stats.types :as stats.types]
- [metabase.metabot.stats.util :as stats.u]
+ [metabase.interestingness.chart.types :as stats.types]
+ [metabase.interestingness.chart.util :as stats.u]
[metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
diff --git a/src/metabase/metabot/stats/outliers.clj b/src/metabase/interestingness/chart/outliers.clj
similarity index 97%
rename from src/metabase/metabot/stats/outliers.clj
rename to src/metabase/interestingness/chart/outliers.clj
index 8953f677dc8..cdd3131b0af 100644
--- a/src/metabase/metabot/stats/outliers.clj
+++ b/src/metabase/interestingness/chart/outliers.clj
@@ -1,11 +1,11 @@
-(ns metabase.metabot.stats.outliers
+(ns metabase.interestingness.chart.outliers
"Modified Z-score based outlier detection for chart analysis.
Uses the Modified Z-score method which is more robust than standard Z-scores
because it uses median and MAD (Median Absolute Deviation) instead of mean
and standard deviation, making it resistant to outliers in the calculation itself."
(:require
- [metabase.metabot.stats.types :as stats.types]
+ [metabase.interestingness.chart.types :as stats.types]
[metabase.util.malli :as mu]
[tech.v3.datatype.argops :as argops]
[tech.v3.datatype.functional :as dfn]))
diff --git a/src/metabase/metabot/stats/repr.clj b/src/metabase/interestingness/chart/repr.clj
similarity index 99%
rename from src/metabase/metabot/stats/repr.clj
rename to src/metabase/interestingness/chart/repr.clj
index 6c8d0f64d63..42511a15c1b 100644
--- a/src/metabase/metabot/stats/repr.clj
+++ b/src/metabase/interestingness/chart/repr.clj
@@ -1,8 +1,8 @@
-(ns metabase.metabot.stats.repr
+(ns metabase.interestingness.chart.repr
"LLM-friendly text representation generator for chart statistics."
(:require
[clojure.string :as str]
- [metabase.metabot.stats.types :as stats.types]
+ [metabase.interestingness.chart.types :as stats.types]
[metabase.util.malli :as mu])
(:import
(java.time LocalDate LocalDateTime ZonedDateTime)
diff --git a/src/metabase/metabot/stats/scatter.clj b/src/metabase/interestingness/chart/scatter.clj
similarity index 93%
rename from src/metabase/metabot/stats/scatter.clj
rename to src/metabase/interestingness/chart/scatter.clj
index 0acff627274..1d3a7a6cb28 100644
--- a/src/metabase/metabot/stats/scatter.clj
+++ b/src/metabase/interestingness/chart/scatter.clj
@@ -1,9 +1,9 @@
-(ns metabase.metabot.stats.scatter
+(ns metabase.interestingness.chart.scatter
"Scatter plot statistics."
(:require
- [metabase.metabot.stats.outliers :as outliers]
- [metabase.metabot.stats.types :as stats.types]
- [metabase.metabot.stats.util :as stats.u]
+ [metabase.interestingness.chart.outliers :as outliers]
+ [metabase.interestingness.chart.types :as stats.types]
+ [metabase.interestingness.chart.util :as stats.u]
[metabase.util.malli :as mu]
[tech.v3.datatype.functional :as dfn]))
diff --git a/src/metabase/metabot/stats/core.clj b/src/metabase/interestingness/chart/stats.clj
similarity index 95%
rename from src/metabase/metabot/stats/core.clj
rename to src/metabase/interestingness/chart/stats.clj
index 0360f02c2d7..f9387861a8c 100644
--- a/src/metabase/metabot/stats/core.clj
+++ b/src/metabase/interestingness/chart/stats.clj
@@ -1,11 +1,11 @@
-(ns metabase.metabot.stats.core
+(ns metabase.interestingness.chart.stats
"Chart type detection and statistics routing."
(:require
- [metabase.metabot.stats.categorical :as categorical]
- [metabase.metabot.stats.histogram :as histogram]
- [metabase.metabot.stats.scatter :as scatter]
- [metabase.metabot.stats.time-series :as time-series]
- [metabase.metabot.stats.types :as stats.types]
+ [metabase.interestingness.chart.categorical :as categorical]
+ [metabase.interestingness.chart.histogram :as histogram]
+ [metabase.interestingness.chart.scatter :as scatter]
+ [metabase.interestingness.chart.time-series :as time-series]
+ [metabase.interestingness.chart.types :as stats.types]
[metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
diff --git a/src/metabase/metabot/stats/time_series.clj b/src/metabase/interestingness/chart/time_series.clj
similarity index 97%
rename from src/metabase/metabot/stats/time_series.clj
rename to src/metabase/interestingness/chart/time_series.clj
index b0906e9ab37..63b4ea6a20b 100644
--- a/src/metabase/metabot/stats/time_series.clj
+++ b/src/metabase/interestingness/chart/time_series.clj
@@ -1,13 +1,13 @@
-(ns metabase.metabot.stats.time-series
+(ns metabase.interestingness.chart.time-series
"Time series statistics computation for chart analysis.
Provides both basic stats (always computed) and deep stats (optional):
- Basic: summary, time range, trend, cumulative detection, outliers
- Deep: volatility, patterns, significant changes, correlations"
(:require
- [metabase.metabot.stats.outliers :as outliers]
- [metabase.metabot.stats.types :as stats.types]
- [metabase.metabot.stats.util :as stats.u]
+ [metabase.interestingness.chart.outliers :as outliers]
+ [metabase.interestingness.chart.types :as stats.types]
+ [metabase.interestingness.chart.util :as stats.u]
[metabase.util.malli :as mu]
[tech.v3.datatype.functional :as dfn]))
diff --git a/src/metabase/metabot/stats/types.clj b/src/metabase/interestingness/chart/types.clj
similarity index 99%
rename from src/metabase/metabot/stats/types.clj
rename to src/metabase/interestingness/chart/types.clj
index 13b7f88779e..0875b8c276d 100644
--- a/src/metabase/metabot/stats/types.clj
+++ b/src/metabase/interestingness/chart/types.clj
@@ -1,4 +1,4 @@
-(ns metabase.metabot.stats.types
+(ns metabase.interestingness.chart.types
"Malli schemas for chart statistics computation."
(:require
[metabase.util.malli.registry :as mr]))
diff --git a/src/metabase/metabot/stats/util.clj b/src/metabase/interestingness/chart/util.clj
similarity index 98%
rename from src/metabase/metabot/stats/util.clj
rename to src/metabase/interestingness/chart/util.clj
index 41d22601820..80c16de3097 100644
--- a/src/metabase/metabot/stats/util.clj
+++ b/src/metabase/interestingness/chart/util.clj
@@ -1,7 +1,7 @@
-(ns metabase.metabot.stats.util
+(ns metabase.interestingness.chart.util
"Shared utilities used across various stats chart types."
(:require
- [metabase.metabot.stats.types :as stats.types]
+ [metabase.interestingness.chart.types :as stats.types]
[metabase.util.malli :as mu]
[tech.v3.datatype.functional :as dfn]))
diff --git a/src/metabase/interestingness/core.clj b/src/metabase/interestingness/core.clj
new file mode 100644
index 00000000000..cf88285b5b1
--- /dev/null
+++ b/src/metabase/interestingness/core.clj
@@ -0,0 +1,27 @@
+(ns metabase.interestingness.core
+ "Public entry point for the interestingness engine.
+
+ This namespace is a thin facade — it only re-exports the user-facing
+ `compute interestingness for X` functions from their role-specific
+ implementation namespaces. If you need scorers, weight profiles, or the
+ composition machinery, depend on the specific namespace directly:
+
+ - [[metabase.interestingness.chart]] — chart-level scoring + stats helpers
+ - [[metabase.interestingness.dimension]] — dimension scorers + canonical weights
+ - [[metabase.interestingness.impl]] — shared composition machinery (internal)"
+ (:require
+ [metabase.interestingness.chart]
+ [metabase.interestingness.chart.repr]
+ [metabase.interestingness.chart.stats]
+ [metabase.interestingness.dimension]
+ [potemkin :as p]))
+
+(p/import-vars
+ [metabase.interestingness.chart
+ chart-interestingness]
+ [metabase.interestingness.dimension
+ dimension-interestingness]
+ [metabase.interestingness.chart.repr
+ generate-representation]
+ [metabase.interestingness.chart.stats
+ compute-chart-stats])
diff --git a/src/metabase/interestingness/dimension.clj b/src/metabase/interestingness/dimension.clj
new file mode 100644
index 00000000000..59af815b402
--- /dev/null
+++ b/src/metabase/interestingness/dimension.clj
@@ -0,0 +1,195 @@
+(ns metabase.interestingness.dimension
+ "Dimension-role interestingness scoring."
+ (:require
+ [metabase.interestingness.impl :as impl]
+ [metabase.util.time :as u.time]))
+
+(set! *warn-on-reflection* true)
+
+(defn- bucket-count-score
+ "Universal quality curve for bucket/group counts. Sweet spot is 10-25 buckets.
+ Returns 0.0 for ≤1 (hard-zero gate for constant fields)."
+ [n]
+ (cond
+ (<= n 1) 0.0
+ (<= n 2) 0.15
+ (<= n 5) (+ 0.15 (* (/ (- n 2.0) 3.0) 0.25)) ; 2→0.15 .. 5→0.4
+ (<= n 10) (+ 0.4 (* (/ (- n 5.0) 5.0) 0.4)) ; 5→0.4 .. 10→0.8
+ (<= n 25) 1.0 ; sweet spot
+ (<= n 50) (- 1.0 (* (/ (- n 25.0) 25.0) 0.3)) ; 25→1.0 .. 50→0.7
+ (<= n 100) (- 0.7 (* (/ (- n 50.0) 50.0) 0.4)) ; 50→0.7 .. 100→0.3
+ :else (max 0.05 (- 0.3 (* 0.05 (Math/log (/ n 100.0)))))))
+
+(def ^:private temporal-bucket-days
+ "Temporal bucketing strategies with approximate days per bucket, coarsest first
+ so `reduce` finds the best match efficiently (coarser = fewer buckets, finer
+ granularities only win if coarser ones overshoot)."
+ [[:year 365.25]
+ [:quarter 91.31]
+ [:month 30.44]
+ [:week 7.0]
+ [:day 1.0]
+ [:hour (/ 1.0 24)]
+ [:minute (/ 1.0 1440)]])
+
+(defn- best-temporal-buckets
+ "Given a day span, find the granularity whose bucket count scores best."
+ [day-span]
+ (if (<= day-span 0)
+ {:granularity :day :count 1 :score 0.0}
+ (reduce (fn [best [gran days-per]]
+ (let [n (/ day-span days-per)
+ s (bucket-count-score n)]
+ (if (> s (:score best)) {:granularity gran :count n :score s} best)))
+ {:granularity :year :count (/ day-span 365.25) :score 0.0}
+ temporal-bucket-days)))
+
+(defn cardinality
+ "Score based on best achievable bucket/group count for visualization.
+ - Temporal fields: evaluates granularities (day/week/month/quarter/year) and picks
+ the one that lands closest to the 10-25 bucket sweet spot.
+ - Numeric fields with high distinct count and a range: auto-binnable, scores high.
+ - All others (categorical, low-cardinality numeric): scores by raw distinct count.
+ Returns 0.0 (hard-zero gate) for constant fields (≤1 distinct value)."
+ [field]
+ (let [distinct-count (get-in field [:fingerprint :global :distinct-count])
+ temporal-fp (get-in field [:fingerprint :type :type/DateTime])
+ numeric-fp (get-in field [:fingerprint :type :type/Number])]
+ (cond
+ ;; No fingerprint data at all
+ (nil? distinct-count)
+ {:score 0.5 :reason "no cardinality data"}
+
+ ;; Constant field — hard-zero gate
+ (<= distinct-count 1)
+ {:score 0.0 :reason "constant field"}
+
+ ;; Temporal with range data → pick best granularity
+ (and temporal-fp (:earliest temporal-fp) (:latest temporal-fp))
+ (let [{:keys [earliest latest]} temporal-fp
+ start (u.time/coerce-to-timestamp earliest)
+ end (u.time/coerce-to-timestamp latest)]
+ (if (and start end)
+ (let [days (u.time/day-diff start end)
+ {:keys [granularity count score]} (best-temporal-buckets days)]
+ {:score score
+ :reason (str (long days) "-day range, ~" (long count) " " (name granularity) " buckets")})
+ {:score 0.5 :reason "unparseable temporal bounds"}))
+
+ ;; Numeric with high cardinality and a nonzero range → auto-binning will produce good buckets
+ (and numeric-fp (> distinct-count 50)
+ (some? (:min numeric-fp)) (some? (:max numeric-fp))
+ (not= (:min numeric-fp) (:max numeric-fp)))
+ {:score 0.9 :reason (str distinct-count " distinct values, auto-binnable")}
+
+ ;; Everything else: score by raw distinct count (categoricals, low-cardinality numerics, etc.)
+ :else
+ {:score (bucket-count-score distinct-count)
+ :reason (str distinct-count " distinct values")})))
+
+(def ^:private dimension-bonus-types
+ "Semantic types that make especially interesting dimensions."
+ [:type/Address
+ :type/Birthdate
+ :type/CancelationTemporal
+ :type/Company
+ :type/CreationTimestamp
+ :type/Currency
+ :type/JoinTemporal
+ :type/Owner
+ :type/Quantity
+ :type/Score
+ :type/Share
+ :type/Subscription
+ :type/Title])
+
+(defn type-bonus
+ "Boost fields with semantic types that tend to produce interesting explorations."
+ [field]
+ (let [semantic-type (:semantic-type field)]
+ (if (some #(isa? semantic-type %) dimension-bonus-types)
+ {:score 1.0 :reason "special type"}
+ {:score 0.5 :reason "no type bonus"})))
+
+(defn temporal-range
+ "Score temporal fields by their date range span. Wider ranges enable richer time-series exploration."
+ [field]
+ (let [temporal-fp (get-in field [:fingerprint :type :type/DateTime])]
+ (if (nil? temporal-fp)
+ {:score 0.5 :reason "not a temporal field"}
+ (let [{:keys [earliest latest]} temporal-fp]
+ (if (or (nil? earliest) (nil? latest))
+ {:score 0.5 :reason "missing temporal bounds"}
+ (let [start (u.time/coerce-to-timestamp earliest)
+ end (u.time/coerce-to-timestamp latest)]
+ (if (or (nil? start) (nil? end))
+ {:score 0.5 :reason "unparseable temporal bounds"}
+ (let [days (u.time/day-diff start end)]
+ (cond
+ (<= days 0) {:score 0.1 :reason "single-point temporal range"}
+ (< days 7) {:score 0.4 :reason (str days " day range")}
+ (< days 30) {:score 0.6 :reason (str days " day range")}
+ (< days 365) {:score 0.8 :reason (str days " day range")}
+ :else {:score 0.95 :reason (str days " day range")})))))))))
+
+(defn text-structure
+ "Penalize text fields that are structured data (JSON, URLs, email, state codes), mostly
+ blank, or free-form long text. These make poor breakout candidates.
+
+ Email / state percentage checks are a safety net for fields the classifier missed:
+ normally they get `:type/Email` / `:type/State` semantic types which are handled by
+ `type-bonus`, but for borderline or misclassified fields the raw percentages still
+ catch them here."
+ [field]
+ (let [text-fp (get-in field [:fingerprint :type :type/Text])]
+ (if (nil? text-fp)
+ {:score 0.5 :reason "not a text field"}
+ (let [{:keys [percent-json percent-url percent-email percent-state average-length percent-blank]} text-fp]
+ (cond
+ (and (some? percent-blank) (> percent-blank 0.8))
+ {:score 0.1 :reason (str (long (* 100 percent-blank)) "% blank/empty")}
+
+ (and (some? percent-json) (> percent-json 0.9))
+ {:score 0.1 :reason (str (long (* 100 percent-json)) "% JSON content")}
+
+ (and (some? percent-url) (> percent-url 0.9))
+ {:score 0.15 :reason (str (long (* 100 percent-url)) "% URL content")}
+
+ (and (some? percent-email) (> percent-email 0.9))
+ {:score 0.2 :reason (str (long (* 100 percent-email)) "% email content (likely PII, poor breakout)")}
+
+ (and (some? percent-state) (> percent-state 0.9))
+ {:score 0.4 :reason (str (long (* 100 percent-state)) "% state codes (a map visualization fits better)")}
+
+ (and (some? average-length) (> average-length 100))
+ {:score 0.05 :reason (str "avg length " (long average-length) " chars")}
+
+ (and (some? average-length) (> average-length 50))
+ {:score 0.2 :reason (str "avg length " (long average-length) " chars")}
+
+ :else
+ {:score 0.8 :reason "short structured text"})))))
+
+;;; -------------------------------------------------- Weight profiles --------------------------------------------------
+
+(def canonical-dimension-weights
+ "Canonical weight profile for scoring a field as a *dimension* (breakout column).
+ Persisted as `dimension_interestingness` on metabase_field. Rewards structural
+ cleanliness, good bucket counts, category/temporal types, and balanced distributions."
+ {impl/type-penalty 0.30
+ cardinality 0.20
+ impl/nullness 0.10
+ type-bonus 0.10
+ temporal-range 0.05
+ text-structure 0.05
+ impl/distribution-shape 0.15
+ impl/numeric-variance 0.05})
+
+(defn dimension-interestingness
+ "Return the canonical `dimension_interestingness` score for `field`.
+
+ Accepts either a raw DB-style field map with snake_case keys or a normalized
+ field map with kebab-case keys. Returns a double in [0.0, 1.0]."
+ [field]
+ (:score (impl/score-field canonical-dimension-weights
+ (impl/normalize-field field))))
diff --git a/src/metabase/interestingness/impl.clj b/src/metabase/interestingness/impl.clj
new file mode 100644
index 00000000000..aa616c229a4
--- /dev/null
+++ b/src/metabase/interestingness/impl.clj
@@ -0,0 +1,232 @@
+(ns metabase.interestingness.impl
+ "Shared composition machinery and role-independent scorers for the interestingness
+ engine.
+
+ Scorer contract: `(fn [field]) -> {:score double, :reason string}`
+ - field: map of field/dimension metadata (kebab-case keys)
+ - score: double in [0.0, 1.0] where 0.0 = uninteresting, 1.0 = very interesting
+ - reason: short human-readable explanation
+
+ Score semantics:
+ 0.0 Hard exclude (e.g. PK, hidden field)
+ 0.01-0.29 Very low value, likely noise
+ 0.30-0.49 Below average
+ 0.50 Neutral / unknown (missing metadata)
+ 0.51-0.74 Decent, typical useful field
+ 0.75-1.0 High value for exploration"
+ (:require
+ [clojure.set :as set]))
+
+(set! *warn-on-reflection* true)
+
+;;; -------------------------------------------------- Shared scorers --------------------------------------------------
+
+(defn type-penalty
+ "Hard-zero fields whose data values carry no exploratory meaning regardless of the
+ role they're being scored for (dimension or measure):
+ - `:type/PK`: opaque row identifiers
+ - `:type/Collection` / `:type/Structured`: structured blobs (JSON, XML, arrays,
+ dictionaries, text-stored serialized JSON) — not groupable or aggregatable
+ - `:type/UpdatedTimestamp` / `:type/DeletionTimestamp`: audit fields that describe
+ the record, not the entity"
+ [field]
+ (let [semantic-type (:semantic-type field)]
+ (cond
+ (nil? semantic-type) {:score 1.0 :reason "no semantic type"}
+ (isa? semantic-type :type/PK) {:score 0.0 :reason "primary key"}
+ (isa? semantic-type :type/Collection) {:score 0.0 :reason "structured blob"}
+ (isa? semantic-type :type/Structured) {:score 0.0 :reason "structured blob"}
+ (isa? semantic-type :type/UpdatedTimestamp) {:score 0.0 :reason "updated timestamp"}
+ (isa? semantic-type :type/DeletionTimestamp) {:score 0.0 :reason "deletion timestamp"}
+ :else {:score 1.0 :reason "no type penalty"})))
+
+(defn nullness
+ "Linear penalty based on null percentage. Mostly-null fields are noise whether being
+ used as a dimension (empty bars) or a measure (nothing to aggregate)."
+ [field]
+ (if-let [nil-pct (get-in field [:fingerprint :global :nil%])]
+ {:score (- 1.0 nil-pct)
+ :reason (if (> nil-pct 0.95)
+ "mostly null"
+ (str (long (* 100 nil-pct)) "% null"))}
+ {:score 0.5 :reason "no null data"}))
+
+(defn numeric-variance
+ "Score numeric fields by their statistical spread. Zero-variance fields are
+ uninteresting regardless of role — nothing to show as a dim (one bucket) or
+ as a measure (SUM of a constant)."
+ [field]
+ (let [num-fp (get-in field [:fingerprint :type :type/Number])]
+ (if (nil? num-fp)
+ {:score 0.5 :reason "not a numeric field"}
+ (let [{:keys [sd q1 q3 avg], mn :min, mx :max} num-fp]
+ (cond
+ (and (some? sd) (zero? sd))
+ {:score 0.0 :reason "zero variance"}
+
+ (and (some? q1) (some? q3) (== q1 q3))
+ {:score 0.1 :reason "no interquartile spread"}
+
+ (and (some? sd) (some? avg) (not (zero? avg)))
+ (let [cv (abs (/ sd avg))
+ score (min 1.0 (max 0.2 (* 0.5 (+ 1.0 (Math/log (+ 1.0 cv))))))]
+ {:score score :reason (str "coefficient of variation: " (double cv))})
+
+ (and (some? q1) (some? q3) (some? mn) (some? mx) (not (== mn mx)))
+ (let [iqr (- q3 q1)
+ rng (- mx mn)
+ ratio (/ iqr rng)
+ score (min 1.0 (max 0.2 (+ 0.3 (* 0.7 ratio))))]
+ {:score score :reason (str "IQR ratio: " (double ratio))})
+
+ :else
+ {:score 0.5 :reason "insufficient numeric stats"})))))
+
+(defn- skewness-score
+ [skewness]
+ (let [abs-sk (abs (double skewness))]
+ (cond
+ (< abs-sk 0.5) {:score 1.0 :reason (str "symmetric distribution (skewness " (format "%.2f" (double skewness)) ")")}
+ (< abs-sk 1.0) {:score 0.85 :reason (str "slightly skewed (skewness " (format "%.2f" (double skewness)) ")")}
+ (< abs-sk 2.0) {:score 0.6 :reason (str "moderately skewed (skewness " (format "%.2f" (double skewness)) ")")}
+ (< abs-sk 4.0) {:score 0.3 :reason (str "heavily skewed (skewness " (format "%.2f" (double skewness)) ")")}
+ :else {:score 0.15 :reason (str "extremely skewed (skewness " (format "%.2f" (double skewness)) ")")})))
+
+(defn- kurtosis-score
+ [kurtosis]
+ (let [k (double kurtosis)]
+ (cond
+ (< -1.0 k 3.0) {:score 1.0 :reason (str "normal-ish tails (kurtosis " (format "%.2f" k) ")")}
+ (< -2.0 k 7.0) {:score 0.7 :reason (str "moderately heavy tails (kurtosis " (format "%.2f" k) ")")}
+ (< -3.0 k 15.0) {:score 0.4 :reason (str "heavy tails / outlier-dominated (kurtosis " (format "%.2f" k) ")")}
+ :else {:score 0.15 :reason (str "extreme tail weight (kurtosis " (format "%.2f" k) ")")})))
+
+(defn- mode-dominance-score
+ [mode-fraction]
+ (cond
+ (>= mode-fraction 0.95) {:score 0.05 :reason (str (long (* 100 mode-fraction)) "% single value (near-constant)")}
+ (>= mode-fraction 0.80) {:score 0.2 :reason (str (long (* 100 mode-fraction)) "% single value (heavily dominated)")}
+ (>= mode-fraction 0.60) {:score 0.5 :reason (str (long (* 100 mode-fraction)) "% single value")}
+ (>= mode-fraction 0.40) {:score 0.75 :reason (str (long (* 100 mode-fraction)) "% single value")}
+ :else {:score 1.0 :reason (str (long (* 100 mode-fraction)) "% single value (well-distributed)")}))
+
+(defn- top-3-concentration-score
+ [top-3-fraction]
+ (cond
+ (>= top-3-fraction 0.99) {:score 0.1 :reason (str (long (* 100 top-3-fraction)) "% in top 3 (effectively 3 values)")}
+ (>= top-3-fraction 0.95) {:score 0.3 :reason (str (long (* 100 top-3-fraction)) "% in top 3")}
+ :else {:score 1.0 :reason (str (long (* 100 top-3-fraction)) "% in top 3")}))
+
+(defn- zero-dominance-score
+ [zero-fraction]
+ (cond
+ (>= zero-fraction 0.95) {:score 0.05 :reason (str (long (* 100 zero-fraction)) "% zeros (sparse/empty)")}
+ (>= zero-fraction 0.80) {:score 0.2 :reason (str (long (* 100 zero-fraction)) "% zeros")}
+ (>= zero-fraction 0.60) {:score 0.6 :reason (str (long (* 100 zero-fraction)) "% zeros")}
+ :else {:score 1.0 :reason (str (long (* 100 zero-fraction)) "% zeros")}))
+
+(defn distribution-shape
+ "Score fields by the shape of their value distribution. Penalizes fields where:
+ - A single value dominates (high mode-fraction): near-constant, poor for visualization
+ - Top-3 values cover nearly everything (few real categories)
+ - Zero values dominate (numeric counters / sparse data)
+ - Distribution is heavily skewed or has extreme tail weight (kurtosis)
+
+ Works on numeric, text, and temporal fields, and is role-neutral: a flat/skewed
+ distribution is boring both as a breakout and as an aggregation target. Returns 0.5
+ (neutral) when no distribution data is available. Signals are combined worst-of-N:
+ any single strong boringness signal dominates the score."
+ [field]
+ (let [num-fp (get-in field [:fingerprint :type :type/Number])
+ text-fp (get-in field [:fingerprint :type :type/Text])
+ date-fp (get-in field [:fingerprint :type :type/DateTime])
+ mode-frac (or (:mode-fraction num-fp) (:mode-fraction text-fp) (:mode-fraction date-fp))
+ top3 (or (:top-3-fraction num-fp) (:top-3-fraction text-fp) (:top-3-fraction date-fp))
+ skewness (or (:skewness num-fp) (:skewness date-fp))
+ kurtosis (:excess-kurtosis num-fp)
+ zero-frac (:zero-fraction num-fp)
+ sub-scores (cond-> []
+ (some? mode-frac) (conj (mode-dominance-score mode-frac))
+ (some? top3) (conj (top-3-concentration-score top3))
+ (some? zero-frac) (conj (zero-dominance-score zero-frac))
+ (some? skewness) (conj (skewness-score skewness))
+ (some? kurtosis) (conj (kurtosis-score kurtosis)))]
+ (if (empty? sub-scores)
+ {:score 0.5 :reason "no distribution data"}
+ (let [worst (apply min-key :score sub-scores)]
+ {:score (:score worst)
+ :reason (:reason worst)}))))
+
+;;; -------------------------------------------------- Composition --------------------------------------------------
+
+(defn score-field
+ "Score a single field using weighted scorers. Returns:
+ {:score double ;; weighted average in [0.0, 1.0]
+ :scores {scorer-fn {:score double, :reason string}} ;; per-scorer breakdown
+ :field field} ;; the input field, passed through
+
+ If any scorer with nonzero weight returns exactly 0.0, the final score is clamped
+ to at most 0.1. This lets hard signals act as effective gates regardless of what
+ other scorers return."
+ [scorer-weight-map field]
+ (let [total-weight (reduce + 0.0 (vals scorer-weight-map))
+ results (reduce-kv
+ (fn [acc scorer weight]
+ (let [{:keys [score] :as result} (scorer field)]
+ (-> acc
+ (update :weighted-sum + (* weight score))
+ (update :has-hard-zero? #(or % (and (pos? weight) (zero? score))))
+ (assoc-in [:scores scorer] result))))
+ {:weighted-sum 0.0 :scores {} :has-hard-zero? false}
+ scorer-weight-map)
+ raw-score (if (pos? total-weight)
+ (/ (:weighted-sum results) total-weight)
+ 0.5)
+ final-score (if (:has-hard-zero? results)
+ (min raw-score 0.1)
+ raw-score)]
+ {:score final-score
+ :scores (:scores results)
+ :field field}))
+
+(defn compose
+ "Combine multiple scorers with weights into a single scorer function.
+ Returns `(fn [field]) -> {:score double, :scores map, :field map}`."
+ [scorer-weight-map]
+ (fn [field] (score-field scorer-weight-map field)))
+
+(defn apply-cutoff
+ "Filter a sequence of scored field results, keeping only those at or above `threshold`."
+ [threshold scored-fields]
+ (filter #(>= (:score %) threshold) scored-fields))
+
+(defn score-and-filter
+ "Score all fields with the given scorer-weight-map and return only those at or above `cutoff`.
+ Results are sorted by score descending."
+ [scorer-weight-map fields cutoff]
+ (->> fields
+ (map #(score-field scorer-weight-map %))
+ (apply-cutoff cutoff)
+ (sort-by :score >)))
+
+;;; -------------------------------------------------- Field Normalization --------------------------------------------------
+
+(def ^:private snake->kebab-keys
+ "Keys that need conversion from snake_case (raw DB Field maps) to kebab-case (scorer input)."
+ {:semantic_type :semantic-type
+ :base_type :base-type
+ :effective_type :effective-type
+ :visibility_type :visibility-type})
+
+(defn normalize-field
+ "Convert a snake_case field map (as used in x-rays/raw DB) to the kebab-case shape scorers expect.
+ Keys not in the rename map are passed through unchanged (e.g., :fingerprint)."
+ [field]
+ (set/rename-keys field snake->kebab-keys))
+
+(defn score-raw-field
+ "Score a snake_case field map by normalizing it first. Returns the same shape as `score-field`
+ but with the original (unnormalized) field in `:field`."
+ [scorer-weight-map field]
+ (let [result (score-field scorer-weight-map (normalize-field field))]
+ (assoc result :field field)))
diff --git a/src/metabase/lib/schema/metadata/fingerprint.cljc b/src/metabase/lib/schema/metadata/fingerprint.cljc
index 7f28f2a49e7..63b7c6ca851 100644
--- a/src/metabase/lib/schema/metadata/fingerprint.cljc
+++ b/src/metabase/lib/schema/metadata/fingerprint.cljc
@@ -22,12 +22,17 @@
"Schema for fingerprint information for Fields deriving from `:type/Number`."
[:map
{:decode/normalize lib.schema.common/normalize-map-no-kebab-case}
- [:min {:optional true} [:maybe number?]]
- [:max {:optional true} [:maybe number?]]
- [:avg {:optional true} [:maybe number?]]
- [:q1 {:optional true} [:maybe number?]]
- [:q3 {:optional true} [:maybe number?]]
- [:sd {:optional true} [:maybe number?]]])
+ [:min {:optional true} [:maybe number?]]
+ [:max {:optional true} [:maybe number?]]
+ [:avg {:optional true} [:maybe number?]]
+ [:q1 {:optional true} [:maybe number?]]
+ [:q3 {:optional true} [:maybe number?]]
+ [:sd {:optional true} [:maybe number?]]
+ [:skewness {:optional true} [:maybe number?]]
+ [:excess-kurtosis {:optional true} [:maybe number?]]
+ [:mode-fraction {:optional true} [:maybe [:ref ::percent]]]
+ [:top-3-fraction {:optional true} [:maybe [:ref ::percent]]]
+ [:zero-fraction {:optional true} [:maybe [:ref ::percent]]]])
(mr/def ::fingerprint.text
"Schema for fingerprint information for Fields deriving from `:type/Text`."
@@ -37,14 +42,24 @@
[:percent-url {:optional true} [:maybe [:ref ::percent]]]
[:percent-email {:optional true} [:maybe [:ref ::percent]]]
[:percent-state {:optional true} [:maybe [:ref ::percent]]]
- [:average-length {:optional true} [:maybe number?]]])
+ [:percent-blank {:optional true} [:maybe [:ref ::percent]]]
+ [:average-length {:optional true} [:maybe number?]]
+ [:min-length {:optional true} [:maybe number?]]
+ [:max-length {:optional true} [:maybe number?]]
+ [:mode-fraction {:optional true} [:maybe [:ref ::percent]]]
+ [:top-3-fraction {:optional true} [:maybe [:ref ::percent]]]])
(mr/def ::fingerprint.temporal
"Schema for fingerprint information for Fields deriving from `:type/Temporal`."
[:map
{:decode/normalize lib.schema.common/normalize-map-no-kebab-case}
- [:earliest {:optional true} [:maybe :string]]
- [:latest {:optional true} [:maybe :string]]])
+ [:earliest {:optional true} [:maybe :string]]
+ [:latest {:optional true} [:maybe :string]]
+ [:skewness {:optional true} [:maybe number?]]
+ [:mode-fraction {:optional true} [:maybe [:ref ::percent]]]
+ [:top-3-fraction {:optional true} [:maybe [:ref ::percent]]]
+ [:weekday-distribution {:optional true} [:maybe [:sequential [:ref ::percent]]]]
+ [:hour-distribution {:optional true} [:maybe [:sequential [:ref ::percent]]]]])
(mr/def ::fingerprint.type-specific
"Schema for type-specific fingerprint information."
diff --git a/src/metabase/metabot/tools/analyze_chart.clj b/src/metabase/metabot/tools/analyze_chart.clj
index 0312d62693d..d406a1c3fef 100644
--- a/src/metabase/metabot/tools/analyze_chart.clj
+++ b/src/metabase/metabot/tools/analyze_chart.clj
@@ -5,9 +5,8 @@
series data in chart_configs, which is seeded into agent memory during
initialization. This avoids re-executing queries."
(:require
+ [metabase.interestingness.core :as interestingness]
[metabase.metabot.scope :as scope]
- [metabase.metabot.stats.core :as stats.core]
- [metabase.metabot.stats.repr :as stats.repr]
[metabase.metabot.tools.shared :as shared]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
@@ -68,12 +67,12 @@ Do not use headers (##). Do not list statistics. Do not analyze series separatel
(resource/stack-resource-context
(let [{:keys [timeline_events title display_type]} chart-config
opts {:deep? (if (nil? deep) true (boolean deep))}
- stats (stats.core/compute-chart-stats chart-config opts)
+ stats (interestingness/compute-chart-stats chart-config opts)
context {:title title
:display-type display_type
:stats stats
:timeline-events timeline_events}
- representation (stats.repr/generate-representation context)]
+ representation (interestingness/generate-representation context)]
{:output (str representation "\n\n---\n\n" interpretation-guidance)}))
{:output (str "Chart config not found: " chart_config_id
". Available chart configs can be found in the viewing context.")})
diff --git a/src/metabase/sync/analyze.clj b/src/metabase/sync/analyze.clj
index 9b69cce14c0..9aa73c07eb7 100644
--- a/src/metabase/sync/analyze.clj
+++ b/src/metabase/sync/analyze.clj
@@ -6,6 +6,7 @@
(:require
[metabase.sync.analyze.classify :as classify]
[metabase.sync.analyze.fingerprint :as sync.fingerprint]
+ [metabase.sync.analyze.interestingness :as sync.interestingness]
[metabase.sync.interface :as i]
[metabase.sync.util :as sync-util]
[metabase.util :as u]
@@ -75,6 +76,7 @@
(sync.fingerprint/fingerprint-table! table)
(classify/classify-fields! table)
(classify/classify-table! table)
+ (sync.interestingness/score-fields! table)
(update-fields-last-analyzed! table))
(defn- maybe-log-progress [progress-bar-fn]
@@ -95,6 +97,10 @@
(format "Total number of tables classified %d, %d updated"
total-tables tables-classified))
+(defn- interestingness-summary [{:keys [fields-scored fields-failed]}]
+ (format "Interestingness scored %d fields, %d failed"
+ fields-scored fields-failed))
+
(defn- make-analyze-steps [log-fn]
[(sync-util/create-sync-step "fingerprint-fields"
#(sync.fingerprint/fingerprint-fields-for-db! % log-fn)
@@ -104,7 +110,10 @@
classify-fields-summary)
(sync-util/create-sync-step "classify-tables"
#(classify/classify-tables-for-db! % log-fn)
- classify-tables-summary)])
+ classify-tables-summary)
+ (sync-util/create-sync-step "score-interestingness"
+ #(sync.interestingness/score-fields-for-db! % log-fn)
+ interestingness-summary)])
(mu/defn analyze-db!
"Perform in-depth analysis on the data for all Tables in a given `database`. This is dependent on what each database
@@ -112,7 +121,7 @@
`:last_analyzed` value for each affected Field."
[database :- i/DatabaseInstance]
(sync-util/sync-operation :analyze database (format "Analyze data for %s" (sync-util/name-for-logging database))
- (sync-util/with-emoji-progress-bar [emoji-progress-bar (inc (* 3 (sync-util/sync-tables-count database)))]
+ (sync-util/with-emoji-progress-bar [emoji-progress-bar (inc (* 4 (sync-util/sync-tables-count database)))]
(u/prog1 (sync-util/run-sync-operation "analyze" database (make-analyze-steps (maybe-log-progress emoji-progress-bar)))
(update-fields-last-analyzed-for-db! database)))))
diff --git a/src/metabase/sync/analyze/interestingness.clj b/src/metabase/sync/analyze/interestingness.clj
new file mode 100644
index 00000000000..18d3c391215
--- /dev/null
+++ b/src/metabase/sync/analyze/interestingness.clj
@@ -0,0 +1,63 @@
+(ns metabase.sync.analyze.interestingness
+ "Analysis sub-step that computes a canonical dimension-interestingness score for each
+ field and persists it on `metabase_field.dimension_interestingness`.
+
+ Runs after fingerprinting and classification so that scorers have both the statistical
+ fingerprint and the inferred semantic type available. Scores are recomputed whenever a
+ field is re-fingerprinted; there is no separate version tracking."
+ (:require
+ [metabase.interestingness.core :as interestingness]
+ [metabase.sync.analyze.fingerprint :as sync.fingerprint]
+ [metabase.sync.interface :as i]
+ [metabase.sync.util :as sync-util]
+ [metabase.util :as u]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(mu/defn- score-and-save!
+ "Score a single field's dimension role and persist the composite score."
+ [field :- i/FieldInstance]
+ (sync-util/with-error-handling (format "Error scoring interestingness for %s" (sync-util/name-for-logging field))
+ (let [dim-score (interestingness/dimension-interestingness field)]
+ (t2/update! :model/Field (u/the-id field)
+ {:dimension_interestingness dim-score}))))
+
+(mu/defn- fields-to-score :- [:maybe [:sequential i/FieldInstance]]
+ "Return Fields in `table` with fresh fingerprints that haven't completed analysis yet."
+ [table :- i/TableInstance]
+ (seq (apply t2/select :model/Field
+ :table_id (u/the-id table)
+ :active true
+ :visibility_type [:not-in ["sensitive" "retired"]]
+ (reduce concat [] (sync.fingerprint/incomplete-analysis-kvs)))))
+
+(mu/defn score-fields!
+ "Score interestingness for all qualifying Fields in `table`."
+ [table :- i/TableInstance]
+ (if-let [fields (fields-to-score table)]
+ (do
+ (log/debugf "Scoring interestingness for %d fields in %s" (count fields) (sync-util/name-for-logging table))
+ (reduce (fn [stats field]
+ (let [result (score-and-save! field)]
+ (if (instance? Exception result)
+ (update stats :fields-failed inc)
+ (update stats :fields-scored inc))))
+ {:fields-scored 0 :fields-failed 0}
+ fields))
+ {:fields-scored 0 :fields-failed 0}))
+
+(mu/defn score-fields-for-db!
+ "Score interestingness for all qualifying Fields in `database`."
+ [database :- i/DatabaseInstance
+ log-progress-fn]
+ (let [tables (sync-util/reducible-sync-tables database)]
+ (transduce (map (fn [table]
+ (let [result (score-fields! table)]
+ (log-progress-fn "score-interestingness" table)
+ result)))
+ (partial merge-with +)
+ {:fields-scored 0 :fields-failed 0}
+ tables)))
diff --git a/src/metabase/sync/interface.clj b/src/metabase/sync/interface.clj
index 2a88fa72a95..ffbb748001a 100644
--- a/src/metabase/sync/interface.clj
+++ b/src/metabase/sync/interface.clj
@@ -192,7 +192,8 @@
2 #{:type/Number}
3 #{:type/DateTime}
4 #{:type/*}
- 5 #{:type/Text}})
+ 5 #{:type/Text}
+ 6 #{:type/Number :type/Text :type/DateTime}})
(def ^:dynamic ^Long *latest-fingerprint-version*
"The newest (highest-numbered) version of our Field fingerprints."
diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.clj
index e6240724c44..4e6d0390044 100644
--- a/src/metabase/sync/util.clj
+++ b/src/metabase/sync/util.clj
@@ -319,7 +319,9 @@
"😎"]) ; smiling face with sunglasses
(defn- percent-done->emoji [percent-done]
- (progress-emoji (int (math/round (* percent-done (dec (count progress-emoji)))))))
+ (let [last-idx (dec (count progress-emoji))
+ idx (int (math/round (* percent-done last-idx)))]
+ (progress-emoji (max 0 (min last-idx idx)))))
(defn emoji-progress-bar
"Create a string that shows progress for something, e.g. a database sync process.
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index 99e89050417..834ae746098 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -460,7 +460,7 @@
:description :display_name :effective_type :has_field_values :is_defective_duplicate
:json_unfolding :name :nfc_path :points_of_interest :position :preview_display :semantic_type :settings
:unique_field_helper :visibility_type]
- :skip [:fingerprint :fingerprint_version :last_analyzed]
+ :skip [:dimension_interestingness :fingerprint :fingerprint_version :last_analyzed]
:transform {:created_at (serdes/date)
:table_id (serdes/fk :model/Table)
:fk_target_field_id (serdes/fk :model/Field)
diff --git a/test/metabase/analyze/fingerprint/fingerprinters_test.clj b/test/metabase/analyze/fingerprint/fingerprinters_test.clj
index 4c2bee9cf02..e93e9f21f98 100644
--- a/test/metabase/analyze/fingerprint/fingerprinters_test.clj
+++ b/test/metabase/analyze/fingerprint/fingerprinters_test.clj
@@ -12,6 +12,15 @@
(set! *warn-on-reflection* true)
+(defn- temporal-core-fields
+ "Return the subset of a temporal fingerprint result containing the historically-asserted fields.
+ We compare only :earliest / :latest (plus :global) to keep tests stable against new distribution
+ stats (skewness, mode-fraction, weekday-distribution, hour-distribution)."
+ [result]
+ {:global (:global result)
+ :type {:type/DateTime (select-keys (get-in result [:type :type/DateTime])
+ [:earliest :latest])}})
+
(deftest fingerprint-temporal-values-test
;; we want to test h2 and postgres, because h2 doesn't
;; support overriding the timezone for a session / report
@@ -24,63 +33,70 @@
:nil% 0.5}
:type {:type/DateTime {:earliest "2013-01-01"
:latest "2018-01-01"}}}
- (transduce identity
- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/DateTime}))
- [#t "2013" nil #t "2018" nil nil #t "2015"])))
+ (temporal-core-fields
+ (transduce identity
+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/DateTime}))
+ [#t "2013" nil #t "2018" nil nil #t "2015"]))))
(testing "handle ChronoLocalDateTime"
(is (= {:global {:distinct-count 2
:nil% 0.0}
:type {:type/DateTime {:earliest "2013-01-01T20:04:00Z"
:latest "2018-01-01T04:04:00Z"}}}
- (transduce identity
- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))
- [(java.time.LocalDateTime/of 2013 01 01 20 04 0 0)
- (java.time.LocalDateTime/of 2018 01 01 04 04 0 0)]))))
+ (temporal-core-fields
+ (transduce identity
+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))
+ [(java.time.LocalDateTime/of 2013 01 01 20 04 0 0)
+ (java.time.LocalDateTime/of 2018 01 01 04 04 0 0)])))))
(testing "handle comparing explicit Instant with ChronoLocalDateTime"
(is (= {:global {:distinct-count 2
:nil% 0.0}
:type {:type/DateTime {:earliest "2007-12-03T10:15:30Z"
:latest "2018-01-01T04:04:00Z"}}}
- (transduce identity
- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))
- [(java.time.Instant/parse "2007-12-03T10:15:30.00Z")
- (java.time.LocalDateTime/of 2018 01 01 04 04 0 0)]))))
+ (temporal-core-fields
+ (transduce identity
+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))
+ [(java.time.Instant/parse "2007-12-03T10:15:30.00Z")
+ (java.time.LocalDateTime/of 2018 01 01 04 04 0 0)])))))
(testing "mixing numbers and strings"
(is (= {:global {:distinct-count 2
:nil% 0.0}
:type {:type/DateTime {:earliest "1970-01-01T00:00:01.234Z"
:latest "2007-12-03T10:15:30Z"}}}
- (transduce identity
- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))
- ["2007-12-03T10:15:30.00Z" 1234]))))
+ (temporal-core-fields
+ (transduce identity
+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))
+ ["2007-12-03T10:15:30.00Z" 1234])))))
(testing "nil temporal values"
(is (= {:global {:distinct-count 1
:nil% 1.0}
:type {:type/DateTime {:earliest nil
:latest nil}}}
- (transduce identity
- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/DateTime}))
- (repeat 10 nil)))))
+ (temporal-core-fields
+ (transduce identity
+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/DateTime}))
+ (repeat 10 nil))))))
(testing "handle all supported types"
(is (= {:global {:distinct-count 5
:nil% 0.0}
:type {:type/DateTime {:earliest "1970-01-01T00:00:01.234Z"
:latest "2020-07-06T20:25:33.36Z"}}}
- (transduce identity
- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))
- [(java.time.LocalDateTime/of 2013 01 01 20 04 0 0) ; LocalDateTime
- 1234 ; int
- 1594067133360 ; long
- "2007-12-03T10:15:30.00Z" ; string
- (java.time.ZonedDateTime/of 2016 01 01 20 04 0 0 java.time.ZoneOffset/UTC)]))))
+ (temporal-core-fields
+ (transduce identity
+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))
+ [(java.time.LocalDateTime/of 2013 01 01 20 04 0 0) ; LocalDateTime
+ 1234 ; int
+ 1594067133360 ; long
+ "2007-12-03T10:15:30.00Z" ; string
+ (java.time.ZonedDateTime/of 2016 01 01 20 04 0 0 java.time.ZoneOffset/UTC)])))))
(testing "we respect effective_type"
(is (= {:global {:distinct-count 2
:nil% 0.0}
:type {:type/DateTime {:earliest "1970-01-01T00:00:01.234Z"
:latest "2007-12-03T10:15:30Z"}}}
- (transduce identity
- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text :effective_type :type/Temporal}))
- ["2007-12-03T10:15:30.00Z" "1970-01-01T00:00:01.234Z"]))))))))))
+ (temporal-core-fields
+ (transduce identity
+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text :effective_type :type/Temporal}))
+ ["2007-12-03T10:15:30.00Z" "1970-01-01T00:00:01.234Z"])))))))))))
(deftest ^:parallel disambiguate-test
(testing "We should correctly disambiguate multiple competing multimethods (DateTime and FK in this case)"
@@ -92,40 +108,46 @@
:nil% 0.0}
:type {:type/DateTime {:earliest "2013-01-01"
:latest "2018-01-01"}}}
- (transduce identity
- (fingerprinters/fingerprinter field)
- [#t "2013" #t "2018" #t "2015"]))))))
+ (temporal-core-fields
+ (transduce identity
+ (fingerprinters/fingerprinter field)
+ [#t "2013" #t "2018" #t "2015"])))))))
(deftest ^:parallel fingerprint-numeric-values-test
(is (= {:global {:distinct-count 3
:nil% 0.0}
- :type {:type/Number {:avg 2.0
- :min 1.0
- :max 3.0
- :q1 1.25
- :q3 2.75
- :sd 1.0}}}
+ :type {:type/Number {:avg 2.0
+ :min 1.0
+ :max 3.0
+ :q1 1.25
+ :q3 2.75
+ :sd 1.0
+ :skewness 0.0
+ :excess-kurtosis nil
+ :mode-fraction (/ 1.0 3.0)
+ :top-3-fraction 1.0
+ :zero-fraction 0.0}}}
(transduce identity
(fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Number}))
[1.0 2.0 3.0])))
(testing "we respect effective_type"
- (is (= {:global {:distinct-count 4, :nil% 0.0},
- :type {:type/Number {:min 1.0, :q1 1.15, :q3 2.15, :max 2.3, :sd 0.6027713773341707, :avg 1.65}}}
- (transduce identity
- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text :effective_type :type/Number}))
- ["1" "2" "1.3" "2.3"]))))
+ (let [result (transduce identity
+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text :effective_type :type/Number}))
+ ["1" "2" "1.3" "2.3"])]
+ (is (= {:distinct-count 4, :nil% 0.0} (:global result)))
+ (is (= {:min 1.0 :q1 1.15 :q3 2.15 :max 2.3 :avg 1.65}
+ (select-keys (get-in result [:type :type/Number])
+ [:min :q1 :q3 :max :avg])))
+ (is (some? (get-in result [:type :type/Number :skewness])))
+ (is (some? (get-in result [:type :type/Number :mode-fraction])))))
(testing "We should robustly survive weird values such as NaN, Infinity, and nil"
- (is (= {:global {:distinct-count 7
- :nil% 0.25}
- :type {:type/Number {:avg 2.0
- :min 1.0
- :max 3.0
- :q1 1.25
- :q3 2.75
- :sd 1.0}}}
- (transduce identity
- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Number}))
- [1.0 2.0 3.0 Double/NaN Double/POSITIVE_INFINITY Double/NEGATIVE_INFINITY nil nil])))))
+ (let [result (transduce identity
+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Number}))
+ [1.0 2.0 3.0 Double/NaN Double/POSITIVE_INFINITY Double/NEGATIVE_INFINITY nil nil])]
+ (is (= {:distinct-count 7, :nil% 0.25} (:global result)))
+ (is (= {:avg 2.0, :min 1.0, :max 3.0, :q1 1.25, :q3 2.75, :sd 1.0}
+ (select-keys (get-in result [:type :type/Number])
+ [:avg :min :max :q1 :q3 :sd]))))))
(deftest ^:parallel fingerprint-string-values-test
(is (= {:global {:distinct-count 5
@@ -134,7 +156,12 @@
:percent-url 0.0
:percent-email 0.0
:percent-state 0.0
- :average-length 6.4}}}
+ :average-length 6.4
+ :min-length 4.0
+ :max-length 9.0
+ :mode-fraction 0.2
+ :top-3-fraction 0.6
+ :percent-blank 0.0}}}
(transduce identity
(fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text}))
["metabase" "more" "like" "metabae" "[1, 2, 3]"])))
@@ -145,7 +172,12 @@
:percent-url 0.0
:percent-email 0.0
:percent-state 0.0
- :average-length 10.6}}}
+ :average-length 10.6
+ :min-length 4.0
+ :max-length 30.0
+ :mode-fraction 0.2
+ :top-3-fraction 0.6
+ :percent-blank 0.0}}}
(transduce identity
(fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text}))
["metabase" "more" "like" "metabae" truncated-json])))))
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index bcc47b60fdf..76972a9ef58 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -1309,23 +1309,34 @@
" VALUES ('22:00'::time, '9:00'::time, 'Beauty Sleep');")])
(mt/with-temp [:model/Database database {:engine :postgres, :details (assoc details :dbname "time_field_test")}]
(sync/sync-database! database)
- (is (= {"start_time" {:global {:distinct-count 1
- :nil% 0.0}
- :type {:type/DateTime {:earliest "22:00:00"
- :latest "22:00:00"}}}
- "end_time" {:global {:distinct-count 1
- :nil% 0.0}
- :type {:type/DateTime {:earliest "09:00:00"
- :latest "09:00:00"}}}
- "reason" {:global {:distinct-count 1
- :nil% 0.0}
- :type {:type/Text {:percent-json 0.0
- :percent-url 0.0
- :percent-email 0.0
- :percent-state 0.0
- :average-length 12.0}}}}
- (t2/select-fn->fn :name :fingerprint :model/Field
- :table_id (t2/select-one-pk :model/Table :db_id (u/the-id database))))))))))
+ (let [fingerprints (t2/select-fn->fn :name :fingerprint :model/Field
+ :table_id (t2/select-one-pk :model/Table :db_id (u/the-id database)))
+ ;; Strip extended interestingness stats — this test covers the core TIME fingerprint
+ ;; shape (#5911), not the interestingness metrics.
+ extended-keys [:hour-distribution :weekday-distribution :skewness
+ :mode-fraction :top-3-fraction
+ :mode-fraction-by-weekday :mode-fraction-by-hour
+ :min-length :max-length :percent-blank]
+ trim-type (fn [fp]
+ (update fp :type
+ (fn [types]
+ (update-vals types #(apply dissoc % extended-keys)))))]
+ (is (= {"start_time" {:global {:distinct-count 1
+ :nil% 0.0}
+ :type {:type/DateTime {:earliest "22:00:00"
+ :latest "22:00:00"}}}
+ "end_time" {:global {:distinct-count 1
+ :nil% 0.0}
+ :type {:type/DateTime {:earliest "09:00:00"
+ :latest "09:00:00"}}}
+ "reason" {:global {:distinct-count 1
+ :nil% 0.0}
+ :type {:type/Text {:percent-json 0.0
+ :percent-url 0.0
+ :percent-email 0.0
+ :percent-state 0.0
+ :average-length 12.0}}}}
+ (update-vals fingerprints trim-type)))))))))
;;; ----------------------------------------------------- Other ------------------------------------------------------
diff --git a/test/metabase/metabot/stats/categorical_test.clj b/test/metabase/interestingness/chart/categorical_test.clj
similarity index 98%
rename from test/metabase/metabot/stats/categorical_test.clj
rename to test/metabase/interestingness/chart/categorical_test.clj
index 18bc184dd41..6eb9bd271b9 100644
--- a/test/metabase/metabot/stats/categorical_test.clj
+++ b/test/metabase/interestingness/chart/categorical_test.clj
@@ -1,8 +1,8 @@
-(ns metabase.metabot.stats.categorical-test
+(ns metabase.interestingness.chart.categorical-test
(:require
[clojure.test :refer :all]
[mb.hawk.assert-exprs.approximately-equal :as =?]
- [metabase.metabot.stats.categorical :as categorical]))
+ [metabase.interestingness.chart.categorical :as categorical]))
(set! *warn-on-reflection* true)
diff --git a/test/metabase/metabot/stats/histogram_test.clj b/test/metabase/interestingness/chart/histogram_test.clj
similarity index 98%
rename from test/metabase/metabot/stats/histogram_test.clj
rename to test/metabase/interestingness/chart/histogram_test.clj
index d34703058b8..1057aff340f 100644
--- a/test/metabase/metabot/stats/histogram_test.clj
+++ b/test/metabase/interestingness/chart/histogram_test.clj
@@ -1,8 +1,8 @@
-(ns metabase.metabot.stats.histogram-test
+(ns metabase.interestingness.chart.histogram-test
(:require
[clojure.test :refer :all]
[mb.hawk.assert-exprs.approximately-equal :as =?]
- [metabase.metabot.stats.histogram :as histogram]))
+ [metabase.interestingness.chart.histogram :as histogram]))
(set! *warn-on-reflection* true)
diff --git a/test/metabase/metabot/stats/outliers_test.clj b/test/metabase/interestingness/chart/outliers_test.clj
similarity index 97%
rename from test/metabase/metabot/stats/outliers_test.clj
rename to test/metabase/interestingness/chart/outliers_test.clj
index 513cea9f1d0..013ff8b3313 100644
--- a/test/metabase/metabot/stats/outliers_test.clj
+++ b/test/metabase/interestingness/chart/outliers_test.clj
@@ -1,8 +1,8 @@
-(ns metabase.metabot.stats.outliers-test
+(ns metabase.interestingness.chart.outliers-test
(:require
[clojure.test :refer :all]
[mb.hawk.assert-exprs.approximately-equal :as =?]
- [metabase.metabot.stats.outliers :as outliers]))
+ [metabase.interestingness.chart.outliers :as outliers]))
(set! *warn-on-reflection* true)
diff --git a/test/metabase/metabot/stats/repr_test.clj b/test/metabase/interestingness/chart/repr_test.clj
similarity index 96%
rename from test/metabase/metabot/stats/repr_test.clj
rename to test/metabase/interestingness/chart/repr_test.clj
index 29cb114e0f8..9a3dd7f0bbc 100644
--- a/test/metabase/metabot/stats/repr_test.clj
+++ b/test/metabase/interestingness/chart/repr_test.clj
@@ -1,13 +1,13 @@
-(ns metabase.metabot.stats.repr-test
+(ns metabase.interestingness.chart.repr-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
- [metabase.metabot.stats.categorical :as categorical]
- [metabase.metabot.stats.core :as stats.core]
- [metabase.metabot.stats.histogram :as histogram]
- [metabase.metabot.stats.repr :as repr]
- [metabase.metabot.stats.scatter :as scatter]
- [metabase.metabot.stats.time-series :as time-series]
+ [metabase.interestingness.chart.categorical :as categorical]
+ [metabase.interestingness.chart.histogram :as histogram]
+ [metabase.interestingness.chart.repr :as repr]
+ [metabase.interestingness.chart.scatter :as scatter]
+ [metabase.interestingness.chart.stats :as stats.core]
+ [metabase.interestingness.chart.time-series :as time-series]
[metabase.util :as u]))
(set! *warn-on-reflection* true)
diff --git a/test/metabase/metabot/stats/scatter_test.clj b/test/metabase/interestingness/chart/scatter_test.clj
similarity index 98%
rename from test/metabase/metabot/stats/scatter_test.clj
rename to test/metabase/interestingness/chart/scatter_test.clj
index 019b2d7be53..9fcf9d641de 100644
--- a/test/metabase/metabot/stats/scatter_test.clj
+++ b/test/metabase/interestingness/chart/scatter_test.clj
@@ -1,8 +1,8 @@
-(ns metabase.metabot.stats.scatter-test
+(ns metabase.interestingness.chart.scatter-test
(:require
[clojure.test :refer :all]
[mb.hawk.assert-exprs.approximately-equal :as =?]
- [metabase.metabot.stats.scatter :as scatter]))
+ [metabase.interestingness.chart.scatter :as scatter]))
(set! *warn-on-reflection* true)
diff --git a/test/metabase/metabot/stats/core_test.clj b/test/metabase/interestingness/chart/stats_test.clj
similarity index 97%
rename from test/metabase/metabot/stats/core_test.clj
rename to test/metabase/interestingness/chart/stats_test.clj
index 8fc7f4e8f77..ff822797518 100644
--- a/test/metabase/metabot/stats/core_test.clj
+++ b/test/metabase/interestingness/chart/stats_test.clj
@@ -1,7 +1,7 @@
-(ns metabase.metabot.stats.core-test
+(ns metabase.interestingness.chart.stats-test
(:require
[clojure.test :refer :all]
- [metabase.metabot.stats.core :as stats.core]))
+ [metabase.interestingness.chart.stats :as stats.core]))
(set! *warn-on-reflection* true)
diff --git a/test/metabase/metabot/stats/time_series_test.clj b/test/metabase/interestingness/chart/time_series_test.clj
similarity index 99%
rename from test/metabase/metabot/stats/time_series_test.clj
rename to test/metabase/interestingness/chart/time_series_test.clj
index 7cec5b0af2e..9aeab74a6c3 100644
--- a/test/metabase/metabot/stats/time_series_test.clj
+++ b/test/metabase/interestingness/chart/time_series_test.clj
@@ -1,10 +1,10 @@
-(ns metabase.metabot.stats.time-series-test
+(ns metabase.interestingness.chart.time-series-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[mb.hawk.assert-exprs.approximately-equal :as =?]
- [metabase.metabot.stats.time-series :as time-series]
- [metabase.metabot.stats.util :as stats.u]))
+ [metabase.interestingness.chart.time-series :as time-series]
+ [metabase.interestingness.chart.util :as stats.u]))
(set! *warn-on-reflection* true)
diff --git a/test/metabase/metabot/stats/util_test.clj b/test/metabase/interestingness/chart/util_test.clj
similarity index 99%
rename from test/metabase/metabot/stats/util_test.clj
rename to test/metabase/interestingness/chart/util_test.clj
index a0c290ed462..83d60646e78 100644
--- a/test/metabase/metabot/stats/util_test.clj
+++ b/test/metabase/interestingness/chart/util_test.clj
@@ -1,8 +1,8 @@
-(ns metabase.metabot.stats.util-test
+(ns metabase.interestingness.chart.util-test
(:require
[clojure.test :refer :all]
[mb.hawk.assert-exprs.approximately-equal :as =?]
- [metabase.metabot.stats.util :as stats.u]))
+ [metabase.interestingness.chart.util :as stats.u]))
(set! *warn-on-reflection* true)
diff --git a/test/metabase/interestingness/chart_test.clj b/test/metabase/interestingness/chart_test.clj
new file mode 100644
index 00000000000..5f613249a36
--- /dev/null
+++ b/test/metabase/interestingness/chart_test.clj
@@ -0,0 +1,75 @@
+(ns metabase.interestingness.chart-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.interestingness.chart :as chart]
+ [metabase.interestingness.core :as interestingness]))
+
+(set! *warn-on-reflection* true)
+
+(defn- time-series-config
+ "Build a time-series chart-config with the given y_values."
+ [ys]
+ {:display_type "line"
+ :title "Test"
+ :series {"s1" {:x {:name "Date" :type "datetime"}
+ :y {:name "Value" :type "number"}
+ :display_name "Value"
+ :x_values (mapv #(format "2024-01-%02d" (inc %)) (range (count ys)))
+ :y_values (mapv double ys)}}})
+
+(defn- categorical-config
+ [categories values]
+ {:display_type "bar"
+ :title "Test"
+ :series {"s1" {:x {:name "Category" :type "string"}
+ :y {:name "Count" :type "number"}
+ :display_name "Count"
+ :x_values (vec categories)
+ :y_values (mapv double values)}}})
+
+(deftest ^:parallel re-exported-from-core-test
+ (testing "chart-interestingness is re-exported from interestingness.core"
+ (is (some? (resolve 'metabase.interestingness.core/chart-interestingness)))
+ (is (number? (interestingness/chart-interestingness (time-series-config (range 20)))))))
+
+(deftest ^:parallel score-in-unit-interval-test
+ (testing "chart-interestingness always returns a value in [0, 1]"
+ (doseq [cfg [(time-series-config (range 30))
+ (time-series-config (repeat 30 5))
+ (categorical-config ["a" "b" "c"] [10 20 30])
+ (categorical-config ["only"] [42])]]
+ (let [score (chart/chart-interestingness cfg)]
+ (is (<= 0.0 score 1.0) (str "score out of range: " score))))))
+
+(deftest ^:parallel flat-measure-is-uninteresting-test
+ (testing "a chart with zero-variance y_values scores 0"
+ (let [cfg (time-series-config (repeat 30 7))]
+ (is (zero? (chart/chart-interestingness cfg))))))
+
+(deftest ^:parallel single-valued-dimension-is-uninteresting-test
+ (testing "a chart with a single x value scores 0"
+ (let [cfg (categorical-config ["only"] [42])]
+ (is (zero? (chart/chart-interestingness cfg))))))
+
+(deftest ^:parallel trend-beats-flat-test
+ (testing "a strongly-increasing time series scores higher than a nearly-flat one"
+ (let [trend (time-series-config (range 30))
+ noisy (time-series-config (map #(+ 100 (rem % 3)) (range 30)))]
+ (is (> (chart/chart-interestingness trend)
+ (chart/chart-interestingness noisy))))))
+
+(deftest ^:parallel outlier-elevates-score-test
+ (testing "injecting a large outlier raises the score vs. a near-constant series"
+ (let [boring (vec (repeat 29 10))
+ with-out (conj (vec (repeat 29 10)) 10000)]
+ (is (> (chart/chart-interestingness (time-series-config with-out))
+ (chart/chart-interestingness (time-series-config boring)))))))
+
+(deftest ^:parallel caller-supplied-stats-test
+ (testing "2-arity form accepts pre-computed stats (no recomputation)"
+ (let [cfg (time-series-config (range 30))
+ stats {:chart-type :time-series
+ :series-count 0
+ :series {}}
+ score (chart/chart-interestingness cfg stats)]
+ (is (<= 0.0 score 1.0)))))
diff --git a/test/metabase/interestingness/core_test.clj b/test/metabase/interestingness/core_test.clj
new file mode 100644
index 00000000000..7b974df713a
--- /dev/null
+++ b/test/metabase/interestingness/core_test.clj
@@ -0,0 +1,25 @@
+(ns metabase.interestingness.core-test
+ (:require
+ [clojure.test :refer [deftest is]]
+ [metabase.interestingness.core :as interestingness]))
+
+;;; Tests here cover only the public facade — the `compute interestingness for X`
+;;; user-facing legos. The underlying composition machinery is tested in
+;;; `metabase.interestingness.impl-test` and the scorers in
+;;; `metabase.interestingness.dimension-test`.
+
+(deftest ^:parallel dimension-interestingness-scores-normalized-field-test
+ (is (>= (interestingness/dimension-interestingness
+ {:semantic-type :type/CreationTimestamp
+ :base-type :type/DateTime
+ :fingerprint {:global {:distinct-count 5000 :nil% 0.0}
+ :type {:type/DateTime {:earliest "2022-01-01"
+ :latest "2024-12-31"}}}})
+ 0.7)))
+
+(deftest ^:parallel dimension-interestingness-accepts-snake-case-test
+ (is (<= (interestingness/dimension-interestingness
+ {:semantic_type :type/PK
+ :base_type :type/Integer
+ :fingerprint {:global {:distinct-count 1000 :nil% 0.0}}})
+ 0.1)))
diff --git a/test/metabase/interestingness/dimension_test.clj b/test/metabase/interestingness/dimension_test.clj
new file mode 100644
index 00000000000..3dafc86cbae
--- /dev/null
+++ b/test/metabase/interestingness/dimension_test.clj
@@ -0,0 +1,129 @@
+(ns metabase.interestingness.dimension-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.interestingness.dimension :as dim]))
+
+;;; -------------------------------------------------- cardinality --------------------------------------------------
+
+(deftest ^:parallel cardinality-test
+ (testing "constant field (1 distinct) scores 0.0"
+ (is (= 0.0 (:score (dim/cardinality {:fingerprint {:global {:distinct-count 1}}})))))
+ (testing "low cardinality (10 = top of ramp to sweet spot) scores high"
+ (let [score (:score (dim/cardinality {:fingerprint {:global {:distinct-count 10}}}))]
+ (is (>= score 0.8))))
+ (testing "moderate cardinality (50 = past sweet spot, declining) scores moderately"
+ (let [score (:score (dim/cardinality {:fingerprint {:global {:distinct-count 50}}}))]
+ (is (>= score 0.7))))
+ (testing "high cardinality scores lower"
+ (let [score (:score (dim/cardinality {:fingerprint {:global {:distinct-count 5000}}}))]
+ (is (< score 0.5))))
+ (testing "missing fingerprint returns 0.5"
+ (is (= 0.5 (:score (dim/cardinality {}))))
+ (is (= 0.5 (:score (dim/cardinality {:fingerprint {}}))))))
+
+;;; -------------------------------------------------- type-bonus --------------------------------------------------
+
+(deftest ^:parallel type-bonus-test
+ (testing "listed bonus types score 1.0"
+ (doseq [sem-type [:type/CreationTimestamp
+ :type/Country ; via :type/Address
+ :type/ZipCode ; via :type/Address
+ :type/Income ; via :type/Currency
+ :type/Price ; via :type/Currency
+ :type/Birthdate
+ :type/Title
+ :type/Quantity
+ :type/Share
+ :type/Score
+ :type/JoinDate ; via :type/JoinTemporal
+ :type/CancelationTimestamp ; via :type/CancelationTemporal
+ :type/Company
+ :type/Subscription
+ :type/Owner]]
+ (is (= 1.0 (:score (dim/type-bonus {:semantic-type sem-type})))
+ (str sem-type " should score 1.0"))))
+ (testing "non-listed types score 0.5"
+ (doseq [sem-type [:type/Category :type/Name :type/Number :type/Text nil]]
+ (is (= 0.5 (:score (dim/type-bonus {:semantic-type sem-type})))
+ (str sem-type " should score 0.5"))))
+ (testing "base-type alone (no semantic-type) does not earn a bonus"
+ (is (= 0.5 (:score (dim/type-bonus {:base-type :type/DateTime}))))
+ (is (= 0.5 (:score (dim/type-bonus {:base-type :type/Boolean})))))
+ (testing "no semantic type returns 0.5"
+ (is (= 0.5 (:score (dim/type-bonus {}))))))
+
+;;; -------------------------------------------------- temporal-range --------------------------------------------------
+
+(deftest ^:parallel temporal-range-test
+ (testing "multi-year range scores high"
+ (let [score (:score (dim/temporal-range
+ {:fingerprint {:type {:type/DateTime {:earliest "2020-01-01" :latest "2024-01-01"}}}}))]
+ (is (>= score 0.9))))
+ (testing "same-day range scores low"
+ (let [score (:score (dim/temporal-range
+ {:fingerprint {:type {:type/DateTime {:earliest "2024-01-01" :latest "2024-01-01"}}}}))]
+ (is (<= score 0.2))))
+ (testing "short range (few days) scores moderate"
+ (let [score (:score (dim/temporal-range
+ {:fingerprint {:type {:type/DateTime {:earliest "2024-01-01" :latest "2024-01-10"}}}}))]
+ (is (> score 0.3))
+ (is (< score 0.7))))
+ (testing "non-temporal field returns 0.5"
+ (is (= 0.5 (:score (dim/temporal-range {})))))
+ (testing "missing bounds returns 0.5"
+ (is (= 0.5 (:score (dim/temporal-range
+ {:fingerprint {:type {:type/DateTime {:earliest nil :latest nil}}}}))))))
+
+;;; -------------------------------------------------- text-structure --------------------------------------------------
+
+(deftest ^:parallel text-structure-test
+ (testing "high JSON percentage scores low"
+ (let [score (:score (dim/text-structure
+ {:fingerprint {:type {:type/Text {:percent-json 0.95}}}}))]
+ (is (<= score 0.15))))
+ (testing "high URL percentage scores low"
+ (let [score (:score (dim/text-structure
+ {:fingerprint {:type {:type/Text {:percent-url 0.95}}}}))]
+ (is (<= score 0.2))))
+ (testing "long average length scores low"
+ (let [score (:score (dim/text-structure
+ {:fingerprint {:type {:type/Text {:average-length 150}}}}))]
+ (is (<= score 0.25))))
+ (testing "moderate length scores low (soft penalty)"
+ (let [score (:score (dim/text-structure
+ {:fingerprint {:type {:type/Text {:average-length 60}}}}))]
+ (is (> score 0.1))
+ (is (< score 0.3))))
+ (testing "very long text (> 100 chars) scores near-zero but isn't a hard-zero"
+ (let [score (:score (dim/text-structure
+ {:fingerprint {:type {:type/Text {:average-length 200}}}}))]
+ (is (< score 0.1))
+ (is (> score 0.0))))
+ (testing "short text scores high"
+ (let [score (:score (dim/text-structure
+ {:fingerprint {:type {:type/Text {:average-length 10}}}}))]
+ (is (>= score 0.7))))
+ (testing "high email percentage scores low (PII safety net for missed classifier)"
+ (let [result (dim/text-structure
+ {:fingerprint {:type {:type/Text {:percent-email 0.95}}}})]
+ (is (<= (:score result) 0.2))
+ (is (re-find #"email" (:reason result)))))
+ (testing "high state percentage is flagged (suggests map viz, not breakout)"
+ (let [result (dim/text-structure
+ {:fingerprint {:type {:type/Text {:percent-state 0.95}}}})]
+ (is (<= (:score result) 0.4))
+ (is (re-find #"state" (:reason result)))))
+ (testing "non-text field returns 0.5"
+ (is (= 0.5 (:score (dim/text-structure {}))))))
+
+;;; -------------------------------------------------- text-structure percent-blank --------------------------------------------------
+
+(deftest ^:parallel text-structure-blank-test
+ (testing "mostly-blank text scores very low"
+ (let [score (:score (dim/text-structure
+ {:fingerprint {:type {:type/Text {:percent-blank 0.9}}}}))]
+ (is (<= score 0.15))))
+ (testing "low percent-blank doesn't trigger the penalty"
+ (let [score (:score (dim/text-structure
+ {:fingerprint {:type {:type/Text {:percent-blank 0.1 :average-length 10}}}}))]
+ (is (>= score 0.7)))))
diff --git a/test/metabase/interestingness/impl_test.clj b/test/metabase/interestingness/impl_test.clj
new file mode 100644
index 00000000000..7adf5415020
--- /dev/null
+++ b/test/metabase/interestingness/impl_test.clj
@@ -0,0 +1,202 @@
+(ns metabase.interestingness.impl-test
+ (:require
+ [clojure.test :refer [are deftest is testing]]
+ [metabase.interestingness.impl :as impl]))
+
+;;; -------------------------------------------------- type-penalty --------------------------------------------------
+
+(deftest ^:parallel type-penalty-test
+ (testing "penalized types score 0.0"
+ (are [sem-type reason-substr]
+ (let [{:keys [score reason]} (impl/type-penalty {:semantic-type sem-type})]
+ (and (= 0.0 score) (re-find (re-pattern reason-substr) reason)))
+ :type/PK "primary key"
+ :type/SerializedJSON "structured blob"
+ :type/JSON "structured blob"
+ :type/XML "structured blob"
+ :type/Array "structured blob"
+ :type/Dictionary "structured blob"
+ :type/UpdatedTimestamp "updated timestamp"
+ :type/DeletionTimestamp "deletion timestamp"))
+
+ (testing "non-penalized types score 1.0 (including FK — x-ray templates use FK columns)"
+ (are [sem-type]
+ (= 1.0 (:score (impl/type-penalty {:semantic-type sem-type})))
+ :type/Category
+ :type/Name
+ :type/CreationTimestamp
+ :type/Number
+ :type/FK
+ nil))
+
+ (testing "nil semantic type scores 1.0"
+ (is (= 1.0 (:score (impl/type-penalty {}))))))
+
+;;; -------------------------------------------------- score-field composition --------------------------------------------------
+
+(defn- constant-scorer [score reason]
+ (fn [_field] {:score score :reason reason}))
+
+(deftest ^:parallel score-field-test
+ (testing "single scorer"
+ (let [result (impl/score-field
+ {(constant-scorer 0.8 "good") 1.0}
+ {:semantic-type :type/Category})]
+ (is (= 0.8 (:score result)))
+ (is (map? (:scores result)))
+ (is (= {:semantic-type :type/Category} (:field result)))))
+
+ (testing "weighted average of two scorers"
+ (let [result (impl/score-field
+ {(constant-scorer 1.0 "high") 0.75
+ (constant-scorer 0.5 "mid") 0.25}
+ {})]
+ (is (= 0.875 (:score result)))))
+
+ (testing "weights don't need to sum to 1"
+ (let [result (impl/score-field
+ {(constant-scorer 1.0 "a") 3.0
+ (constant-scorer 0.5 "b") 1.0}
+ {})]
+ (is (= 0.875 (:score result)))))
+
+ (testing "hard-zero from any scorer clamps total to at most 0.1"
+ (let [result (impl/score-field
+ {(constant-scorer 1.0 "high") 0.75
+ (constant-scorer 0.0 "gate") 0.25}
+ {})]
+ (is (<= (:score result) 0.1))))
+
+ (testing "empty scorer map returns 0.5"
+ (is (= 0.5 (:score (impl/score-field {} {}))))))
+
+(deftest ^:parallel compose-test
+ (testing "compose returns a callable scorer"
+ (let [composed (impl/compose
+ {(constant-scorer 0.8 "a") 1.0
+ (constant-scorer 0.6 "b") 1.0})
+ result (composed {})]
+ (is (= 0.7 (:score result))))))
+
+(deftest ^:parallel apply-cutoff-test
+ (testing "filters below threshold"
+ (let [scored [{:score 0.8 :field :a}
+ {:score 0.3 :field :b}
+ {:score 0.5 :field :c}]]
+ (is (= [{:score 0.8 :field :a} {:score 0.5 :field :c}]
+ (impl/apply-cutoff 0.5 scored)))))
+
+ (testing "empty input returns empty"
+ (is (empty? (impl/apply-cutoff 0.5 [])))))
+
+(deftest ^:parallel score-and-filter-test
+ (testing "scores, filters, and sorts descending"
+ (let [fields [{:name "good"} {:name "bad"} {:name "ok"}]
+ name-scorer (fn [field]
+ (let [len (count (:name field))]
+ {:score (/ len 10.0) :reason (str len " chars")}))
+ results (impl/score-and-filter
+ {name-scorer 1.0} fields 0.25)]
+ (is (= 2 (count results)))
+ (is (= "good" (-> results first :field :name)))
+ (is (>= (-> results first :score) (-> results second :score))))))
+
+;;; -------------------------------------------------- nullness --------------------------------------------------
+
+(deftest ^:parallel nullness-test
+ (testing "0% null scores 1.0"
+ (is (= 1.0 (:score (impl/nullness {:fingerprint {:global {:nil% 0.0}}})))))
+
+ (testing "100% null scores 0.0"
+ (is (= 0.0 (:score (impl/nullness {:fingerprint {:global {:nil% 1.0}}})))))
+
+ (testing "50% null scores 0.5"
+ (is (= 0.5 (:score (impl/nullness {:fingerprint {:global {:nil% 0.5}}})))))
+
+ (testing "missing fingerprint returns 0.5"
+ (is (= 0.5 (:score (impl/nullness {}))))))
+
+;;; -------------------------------------------------- numeric-variance --------------------------------------------------
+
+(deftest ^:parallel numeric-variance-test
+ (testing "zero sd scores 0.0"
+ (is (= 0.0 (:score (impl/numeric-variance
+ {:fingerprint {:type {:type/Number {:sd 0.0 :avg 10.0}}}})))))
+
+ (testing "q1 = q3 scores low"
+ (is (<= (:score (impl/numeric-variance
+ {:fingerprint {:type {:type/Number {:q1 5.0 :q3 5.0}}}}))
+ 0.1)))
+
+ (testing "healthy spread scores high"
+ (let [score (:score (impl/numeric-variance
+ {:fingerprint {:type {:type/Number {:sd 10.0 :avg 50.0 :min 0 :max 100}}}}))]
+ (is (> score 0.3))))
+
+ (testing "non-numeric field returns 0.5"
+ (is (= 0.5 (:score (impl/numeric-variance {}))))
+ (is (= 0.5 (:score (impl/numeric-variance {:fingerprint {:type {:type/Text {}}}}))))))
+
+;;; -------------------------------------------------- distribution-shape --------------------------------------------------
+
+(deftest ^:parallel distribution-shape-test
+ (testing "no distribution data returns 0.5"
+ (is (= 0.5 (:score (impl/distribution-shape {}))))
+ (is (= 0.5 (:score (impl/distribution-shape
+ {:fingerprint {:type {:type/Number {:avg 10 :sd 2}}}})))))
+
+ (testing "symmetric low-dominance distribution scores high"
+ (let [score (:score (impl/distribution-shape
+ {:fingerprint {:type {:type/Number {:skewness 0.1 :mode-fraction 0.1}}}}))]
+ (is (>= score 0.9))))
+
+ (testing "heavy skew penalizes score"
+ (let [score (:score (impl/distribution-shape
+ {:fingerprint {:type {:type/Number {:skewness 3.0 :mode-fraction 0.1}}}}))]
+ (is (<= score 0.5))))
+
+ (testing "high mode-dominance scores very low"
+ (let [score (:score (impl/distribution-shape
+ {:fingerprint {:type {:type/Text {:mode-fraction 0.97}}}}))]
+ (is (<= score 0.1))))
+
+ (testing "80% dominance scores low but not critical"
+ (let [score (:score (impl/distribution-shape
+ {:fingerprint {:type {:type/Text {:mode-fraction 0.85}}}}))]
+ (is (>= score 0.15))
+ (is (<= score 0.3))))
+
+ (testing "combined signals: worst-of-two wins"
+ (let [score (:score (impl/distribution-shape
+ {:fingerprint {:type {:type/Number {:skewness 0.1 :mode-fraction 0.95}}}}))]
+ (is (<= score 0.1))))
+
+ (testing "works on text fields via type/Text fingerprint"
+ (let [score (:score (impl/distribution-shape
+ {:fingerprint {:type {:type/Text {:mode-fraction 0.3}}}}))]
+ (is (>= score 0.9))))
+
+ (testing "extreme excess kurtosis penalizes score"
+ (let [score (:score (impl/distribution-shape
+ {:fingerprint {:type {:type/Number {:excess-kurtosis 20.0}}}}))]
+ (is (<= score 0.2))))
+
+ (testing "near-normal kurtosis scores high"
+ (let [score (:score (impl/distribution-shape
+ {:fingerprint {:type {:type/Number {:excess-kurtosis 0.2}}}}))]
+ (is (>= score 0.9))))
+
+ (testing "high top-3-fraction penalizes score"
+ (let [score (:score (impl/distribution-shape
+ {:fingerprint {:type {:type/Text {:mode-fraction 0.4 :top-3-fraction 0.995}}}}))]
+ (is (<= score 0.15))))
+
+ (testing "high zero-fraction (numeric) penalizes score"
+ (let [score (:score (impl/distribution-shape
+ {:fingerprint {:type {:type/Number {:zero-fraction 0.96}}}}))]
+ (is (<= score 0.1))))
+
+ (testing "temporal mode-fraction penalizes dumping-ground timestamps"
+ (let [score (:score (impl/distribution-shape
+ {:fingerprint {:type {:type/DateTime {:mode-fraction 0.95}}}}))]
+ (is (<= score 0.1)))))
diff --git a/test/metabase/metabot/agent/tool_output_test.clj b/test/metabase/metabot/agent/tool_output_test.clj
index 5721e41d342..eda5615e790 100644
--- a/test/metabase/metabot/agent/tool_output_test.clj
+++ b/test/metabase/metabot/agent/tool_output_test.clj
@@ -11,6 +11,7 @@
[metabase.metabot.tools.metadata :as metadata-tools]
[metabase.metabot.tools.resources :as resource-tools]
[metabase.metabot.tools.search :as search-tools]
+ [metabase.search.core :as search]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.warehouse-schema.models.field-values :as field-values]
@@ -196,6 +197,7 @@
(deftest search-tool-structured-output-formats-correctly-test
(testing "search tool :structured-output formats to clean XML via format-structured-result"
(mt/test-driver :h2
+ (search/init-index! {:force-reset? false :re-populate? true})
(mt/with-current-user (mt/user->id :crowberto)
(let [result (search-tools/search-tool
{:semantic_queries ["orders"]
@@ -206,6 +208,7 @@
(deftest search-tool-with-models-structured-output-test
(testing "search tool with model results formats correctly"
(mt/test-driver :h2
+ (search/init-index! {:force-reset? false :re-populate? true})
(mt/with-current-user (mt/user->id :crowberto)
(let [model-query (-> (lib/query (mt/metadata-provider)
(lib.metadata/table (mt/metadata-provider) (mt/id :orders)))
@@ -215,6 +218,7 @@
:database_id (mt/id)
:name "Searchable Test Model"
:type :model}]
+ (search/init-index! {:force-reset? false :re-populate? true})
(let [result (search-tools/search-tool
{:semantic_queries ["Searchable Test Model"]
:keyword_queries ["Searchable Test Model"]
diff --git a/test/metabase/metabot/tools/field_stats_test.clj b/test/metabase/metabot/tools/field_stats_test.clj
index 29fa7264cc0..ad8502eb618 100644
--- a/test/metabase/metabot/tools/field_stats_test.clj
+++ b/test/metabase/metabot/tools/field_stats_test.clj
@@ -28,24 +28,46 @@
{:entity-type "table", :entity-id people-id, :field-id state-id, :limit 5}))))
(testing "Getting statistics and values for table fields works."
(mt/as-admin
+ ;; Skewness is a derived double whose last few digits vary across JVMs/platforms,
+ ;; so strip it from the exact-equality check and assert approximate value separately.
+ (let [birth-date-result (metabot.tools.field-stats/field-values
+ {:entity-type "table", :entity-id people-id, :field-id birth-date-id, :limit 5})
+ birth-date-skewness (get-in birth-date-result
+ [:structured-output :value_metadata :statistics :skewness])]
+ (is (some? birth-date-skewness))
+ (is (< (abs (- birth-date-skewness -0.00557870227770)) 1e-6)
+ "skewness should be approximately -0.00557870227770"))
(are [table-id field-id value-metadata]
(= {:structured-output {:result-type :field-metadata
:field_id field-id
:value_metadata value-metadata}}
- (metabot.tools.field-stats/field-values
- {:entity-type "table", :entity-id table-id, :field-id field-id, :limit 5}))
+ (let [result (metabot.tools.field-stats/field-values
+ {:entity-type "table", :entity-id table-id, :field-id field-id, :limit 5})]
+ ;; strip skewness to avoid platform-dependent floating-point mismatch
+ (cond-> result
+ (get-in result [:structured-output :value_metadata :statistics :skewness])
+ (update-in [:structured-output :value_metadata :statistics] dissoc :skewness))))
people-id birth-date-id {:statistics
- {:distinct-count 2308
- :percent-null 0.0
- :earliest "1958-04-26"
- :latest "2000-04-03"}}
+ {:distinct-count 2308
+ :percent-null 0.0
+ :earliest "1958-04-26"
+ :latest "2000-04-03"
+ :hour-distribution nil
+ :mode-fraction 0.0012
+ :top-3-fraction 0.0032
+ :weekday-distribution [0.15 0.1304 0.1416 0.1372 0.156 0.1516 0.1332]}}
people-id state-id {:statistics {:distinct-count 49
:percent-null 0.0
:percent-json 0.0
:percent-url 0.0
:percent-email 0.0
:percent-state 1.0
- :average-length 2.0}
+ :average-length 2.0
+ :max-length 2.0
+ :min-length 2.0
+ :mode-fraction 0.0776
+ :top-3-fraction 0.1624
+ :percent-blank 0.0}
:field_values ["AK" "AL" "AR" "AZ" "CA"]}
products-id category-id {:statistics {:distinct-count 4
:percent-null 0.0
@@ -53,7 +75,12 @@
:percent-url 0.0
:percent-email 0.0
:percent-state 0.0
- :average-length 6.375}
+ :average-length 6.375
+ :max-length 9.0
+ :min-length 5.0
+ :mode-fraction 0.27
+ :top-3-fraction 0.79
+ :percent-blank 0.0}
:field_values ["Doohickey" "Gadget" "Gizmo" "Widget"]})))))
(deftest field-values-model-test
diff --git a/test/metabase/sample_data/impl_test.clj b/test/metabase/sample_data/impl_test.clj
index 97c7a75088b..51637a213a3 100644
--- a/test/metabase/sample_data/impl_test.clj
+++ b/test/metabase/sample_data/impl_test.clj
@@ -97,7 +97,12 @@
:percent-url 0.0
:percent-email 0.0
:percent-state 0.0
- :average-length 13.532}}}
+ :average-length 13.532
+ :max-length 23.0
+ :min-length 7.0
+ :mode-fraction 4.0E-4
+ :top-3-fraction 0.0012
+ :percent-blank 0.0}}}
:base_type :type/Text}
(-> (field db "PEOPLE" "NAME")
;; it should be `nil` after sync but get set to `search` by the auto-inference. We only set `list` in
diff --git a/test/metabase/sync/analyze/interestingness_test.clj b/test/metabase/sync/analyze/interestingness_test.clj
new file mode 100644
index 00000000000..1c56f672864
--- /dev/null
+++ b/test/metabase/sync/analyze/interestingness_test.clj
@@ -0,0 +1,30 @@
+(ns metabase.sync.analyze.interestingness-test
+ (:require
+ [clojure.test :refer [deftest is]]
+ [metabase.interestingness.core :as interestingness]
+ [metabase.interestingness.dimension :as dim]))
+
+;;; Smoke tests for the canonical weight profiles. The sync step itself is verified
+;;; end-to-end via `automagic_dashboards` integration tests (which fingerprint + score
+;;; real tables). Here we just pin down the profile shape and directional behavior.
+
+(deftest ^:parallel canonical-dimension-weights-shape-test
+ (is (map? dim/canonical-dimension-weights))
+ (is (every? fn? (keys dim/canonical-dimension-weights)))
+ (is (every? pos? (vals dim/canonical-dimension-weights))))
+
+(deftest ^:parallel dimension-interestingness-kills-pks-test
+ (let [result (interestingness/dimension-interestingness
+ {:semantic_type :type/PK
+ :base_type :type/Integer
+ :fingerprint {:global {:distinct-count 1000 :nil% 0.0}}})]
+ (is (<= result 0.1))))
+
+(deftest ^:parallel dimension-interestingness-rewards-temporal-test
+ (let [result (interestingness/dimension-interestingness
+ {:semantic_type :type/CreationTimestamp
+ :base_type :type/DateTime
+ :fingerprint {:global {:distinct-count 5000 :nil% 0.0}
+ :type {:type/DateTime {:earliest "2022-01-01"
+ :latest "2024-12-31"}}}})]
+ (is (>= result 0.7))))
diff --git a/test/metabase/sync/analyze_test.clj b/test/metabase/sync/analyze_test.clj
index d94b44c4700..892f69414ca 100644
--- a/test/metabase/sync/analyze_test.clj
+++ b/test/metabase/sync/analyze_test.clj
@@ -225,7 +225,7 @@
:model/Field _ (fake-field table)]
(let [results (analyze-table! table)]
(testing "has the steps performed"
- (is (= ["fingerprint-fields" "classify-fields" "classify-tables"]
+ (is (= ["fingerprint-fields" "classify-fields" "classify-tables" "score-interestingness"]
(->> results :steps (map first)))))
(testing "has start and finish times"
(is (seq (select-keys results [:start-time :end-time]))))))))
diff --git a/test/metabase/sync/sync_test.clj b/test/metabase/sync/sync_test.clj
index 035c8f9655c..3f21902771e 100644
--- a/test/metabase/sync/sync_test.clj
+++ b/test/metabase/sync/sync_test.clj
@@ -113,7 +113,8 @@
(assoc :fields (for [field (t2/select :model/Field, :table_id (:id table), {:order-by [:name]})]
(into {} (-> field
(update :fingerprint map?)
- (update :fingerprint_version (complement zero?))))))
+ (update :fingerprint_version (complement zero?))
+ (update :dimension_interestingness double?)))))
tu/boolean-ids-and-timestamps)))
(defn- table-defaults []
@@ -132,24 +133,26 @@
(defn- field-defaults []
(merge
(mt/object-defaults :model/Field)
- {:created_at true
- :fingerprint false
- :fingerprint_version false
- :fk_target_field_id false
+ {:created_at true
+ :fingerprint false
+ :fingerprint_version false
+ :fk_target_field_id false
:database_is_auto_increment false
- :id true
- :last_analyzed false
- :parent_id false
- :position 0
- :json_unfolding false
- :table_id true
- :updated_at true}))
+ :id true
+ :last_analyzed false
+ :parent_id false
+ :position 0
+ :json_unfolding false
+ :table_id true
+ :updated_at true
+ :dimension_interestingness false}))
(defn- field-defaults-with-fingerprint []
(assoc (field-defaults)
- :last_analyzed true
- :fingerprint_version true
- :fingerprint true))
+ :last_analyzed true
+ :fingerprint_version true
+ :fingerprint true
+ :dimension_interestingness true))
(defn- field:movie-id []
(merge
diff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.clj
index 0eee595e35b..9f7d6e2b0e2 100644
--- a/test/metabase/warehouse_schema_rest/api/table_test.clj
+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj
@@ -90,7 +90,7 @@
field
[:created_at :fingerprint :fingerprint_version :fk_target_field_id :id :last_analyzed :updated_at
:database_required :database_is_auto_increment :database_is_pk :database_is_generated :database_is_nullable
- :entity_id])))
+ :entity_id :dimension_interestingness])))
(deftest ^:parallel list-table-test
(testing "GET /api/table"
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index c80ac3e959b..1c1d802424f 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -116,7 +116,7 @@
(select-keys
field
[:updated_at :id :created_at :last_analyzed :fingerprint :fingerprint_version :fk_target_field_id
- :position]))))
+ :position :dimension_interestingness]))))
(defn- card-with-native-query [card-name & {:as kvs}]
(merge