The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Dimension and chart interestiningness calculation functions (#72630)
Edward Pagetcbeb2d4be61a
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 2115d88c8b7..8e9cdf7cb12 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1020,6 +1020,11 @@:api #{metabase.initialization-status.core}:uses #{}}+ interestingness+ {:team "UX West"+ :api #{metabase.interestingness.core}+ :uses #{util}}+internal-stats{:team "UX West":api #{metabase.internal-stats.core}@@ -1223,6 +1228,7 @@configdocumentsdriver+ interestingnesslegacy-mbqlliblib-be@@ -2119,6 +2125,7 @@database-routingdriverevents+ interestingnesslibmodelspremium-featuresdiff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex ba833f95794..2c84837010e 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -116,6 +116,7 @@eventsformatterinitialization-status+ interestingnessinternal-statsllmlogin-historydiff --git a/resources/migrations/061/20260417_interestingness.yaml b/resources/migrations/061/20260417_interestingness.yamlnew file mode 100644index 00000000000..c3620ae5fe2--- /dev/null+++ b/resources/migrations/061/20260417_interestingness.yaml@@ -0,0 +1,16 @@+databaseChangeLog:+ - changeSet:+ id: v61.int3r2+ author: nvoxland+ comment: Add dimension interestingness score to metabase_field+ changes:+ - addColumn:+ tableName: metabase_field+ columns:+ - column:+ name: dimension_interestingness+ type: double precision+ remarks: >-+ Interestingness score (0.0-1.0) for this field when treated as a dimension in a metric.+ constraints:+ nullable: truediff --git a/src/metabase/analyze/fingerprint/fingerprinters.clj b/src/metabase/analyze/fingerprint/fingerprinters.cljindex 8fe350f2725..5f08cc765f9 100644--- a/src/metabase/analyze/fingerprint/fingerprinters.clj+++ b/src/metabase/analyze/fingerprint/fingerprinters.clj@@ -2,6 +2,7 @@"Non-identifying fingerprinters for various field types."(:require[bigml.histogram.core :as hist]+ [clojure.string :as str][java-time.api :as t][kixi.stats.core :as stats][kixi.stats.math :as math]@@ -16,9 +17,9 @@(:import(com.bigml.histogram Histogram)(com.clearspring.analytics.stream.cardinality HyperLogLogPlus)- (java.time ZoneOffset)+ (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZoneOffset ZonedDateTime)(java.time.chrono ChronoLocalDateTime ChronoZonedDateTime)- (java.time.temporal Temporal)))+ (java.time.temporal ChronoField Temporal)))(set! *warn-on-reflection* true)@@ -232,10 +233,119 @@Temporal (->temporal [this] this)java.util.Date (->temporal [this] (t/instant this)))+(def ^:private mode-stats-max-distinct+ "Maximum number of distinct values to track for mode-fraction / top-3-fraction computation.+ Values beyond this cap are counted toward total but not tracked individually. For typical+ 10k-row samples, the true top values will almost always appear in the first 100 distinct+ values seen."+ 100)++(defn- top-n-fraction+ "Given a map of value->count and a total row count, return the sum of the top-n counts+ divided by total. Returns nil if total is not positive."+ [counts total n]+ (when (pos? total)+ (/ (double (reduce + 0.0 (take n (sort > (vals counts)))))+ (double total))))++(defn- mode-stats+ "Reducer that tracks value frequencies (bounded at `mode-stats-max-distinct` entries)+ and, on completion, returns {:mode-fraction, :top-3-fraction}. High mode-fraction+ signals single-value dominance; high top-3-fraction with moderate mode-fraction signals+ bimodal / few-real-categories distributions."+ ([] [{} 0])+ ([[counts total]]+ {:mode-fraction (top-n-fraction counts total 1)+ :top-3-fraction (top-n-fraction counts total 3)})+ ([[counts total] x]+ (cond+ (nil? x) [counts (inc total)]+ (contains? counts x) [(update counts x inc) (inc total)]+ (< (count counts) mode-stats-max-distinct) [(assoc counts x 1) (inc total)]+ :else [counts (inc total)])))++(defn- ->millis-from-epoch+ "Coerce a `java.time.temporal.Temporal` (as produced by `->temporal`) to long epoch millis.+ Returns nil for unsupported types; callers typically wrap with `(keep ->millis-from-epoch)`+ to strip non-coerceable values."+ [t]+ (cond (instance? Instant t) (.toEpochMilli ^Instant t)+ (instance? OffsetDateTime t) (.toEpochMilli (.toInstant ^OffsetDateTime t))+ (instance? ZonedDateTime t) (.toEpochMilli (.toInstant ^ZonedDateTime t))+ (instance? LocalDate t) (recur (t/offset-date-time t (t/local-time 0) (t/zone-offset 0)))+ (instance? LocalDateTime t) (recur (t/offset-date-time t (t/zone-offset 0)))+ (instance? LocalTime t) (recur (t/offset-date-time (t/local-date "1970-01-01") t (t/zone-offset 0)))+ (instance? OffsetTime t) (recur (t/offset-date-time (t/local-date "1970-01-01") t (t/zone-offset t)))+ :else nil))++(defn- temporal->zoned+ "Promote a `Temporal` to a form whose ChronoField support covers DAY_OF_WEEK and+ HOUR_OF_DAY. `Instant` is coerced to UTC; all other types pass through unchanged+ (e.g. `LocalDate` still supports DAY_OF_WEEK; `LocalTime` still supports HOUR_OF_DAY;+ downstream extractors check `isSupported` per-field)."+ ^Temporal [^Temporal t]+ (if (instance? Instant t)+ (.atZone ^Instant t ZoneOffset/UTC)+ t))++(defn- temporal->weekday+ "Return 1-7 for Monday-Sunday, or nil if `t` is nil or the Temporal has no day-of-week."+ [t]+ (when-let [^Temporal z (some-> t temporal->zoned)]+ (when (.isSupported z ChronoField/DAY_OF_WEEK)+ (.get z ChronoField/DAY_OF_WEEK))))++(defn- temporal->hour+ "Return 0-23 for hour-of-day, or nil if `t` is nil or the Temporal has no hour."+ [t]+ (when-let [^Temporal z (some-> t temporal->zoned)]+ (when (.isSupported z ChronoField/HOUR_OF_DAY)+ (.get z ChronoField/HOUR_OF_DAY))))++(defn- bucket-distribution+ "Reducer factory: returns a reducer that bucketizes values via `bucket-fn` into `n-buckets`+ slots and, on completion, returns a vector of per-bucket fractions (summing to 1.0).+ Values where `bucket-fn` returns nil or out-of-range are skipped. Returns nil if no+ valid values were seen."+ [n-buckets bucket-fn]+ (fn+ ([] [(vec (repeat n-buckets 0)) 0])+ ([[buckets total]]+ (when (pos? total)+ (mapv #(/ (double %) (double total)) buckets)))+ ([[buckets total] x]+ (if-let [idx (bucket-fn x)]+ (if (and (<= 0 idx) (< idx n-buckets))+ [(update buckets idx inc) (inc total)]+ [buckets total])+ [buckets total]))))++(defn- weekday-distribution+ "Reducer producing a 7-element vector of per-weekday fractions (Monday=index 0 .. Sunday=6)."+ []+ (bucket-distribution 7 (fn [t]+ (when-let [dow (temporal->weekday t)]+ (dec dow)))))++(defn- hour-distribution+ "Reducer producing a 24-element vector of per-hour fractions (0..23)."+ []+ (bucket-distribution 24 temporal->hour))+(deffingerprinter :type/DateTime((map ->temporal)- (robust-fuse {:earliest earliest- :latest latest})))+ (redux/post-complete+ (robust-fuse {:earliest earliest+ :latest latest+ :skewness ((keep ->millis-from-epoch) stats/skewness)+ :mode-stats ((keep ->millis-from-epoch) mode-stats)+ :weekday-distribution (weekday-distribution)+ :hour-distribution (hour-distribution)})+ (fn [{:keys [mode-stats] :as fused}]+ (-> fused+ (dissoc :mode-stats)+ (assoc :mode-fraction (:mode-fraction mode-stats)+ :top-3-fraction (:top-3-fraction mode-stats)))))))(defn- histogram"Transducer that summarizes numerical data with a histogram."@@ -259,16 +369,22 @@(deffingerprinter :type/Number(redux/post-complete- ((comp (map ->number) (filter u/real-number?)) histogram)- (fn [h]+ ((comp (map ->number) (filter u/real-number?))+ (redux/juxt histogram stats/skewness stats/kurtosis mode-stats (stats/share zero?)))+ (fn [[h sk ku ms zf]](let [{q1 0.25 q3 0.75} (hist/percentiles h 0.25 0.75)](robust-map- :min (hist/minimum h)- :max (hist/maximum h)- :avg (hist/mean h)- :sd (some-> h hist/variance math/sqrt)- :q1 q1- :q3 q3)))))+ :min (hist/minimum h)+ :max (hist/maximum h)+ :avg (hist/mean h)+ :sd (some-> h hist/variance math/sqrt)+ :q1 q1+ :q3 q3+ :skewness sk+ :excess-kurtosis ku+ :mode-fraction (:mode-fraction ms)+ :top-3-fraction (:top-3-fraction ms)+ :zero-fraction zf)))))(defn- valid-serialized-json?"Is x a serialized JSON dictionary or array. Hueristically recognize maps and arrays. Uses the following strategies:@@ -290,11 +406,21 @@((map str) ; we cast to str to support `field-literal` type overwriting:; `[:field-literal "A_NUMBER" :type/Text]` (which still; returns numbers in the result set)- (robust-fuse {:percent-json (stats/share valid-serialized-json?)- :percent-url (stats/share u/url?)- :percent-email (stats/share u/email?)- :percent-state (stats/share u/state?)- :average-length ((map count) stats/mean)})))+ (redux/post-complete+ (robust-fuse {:percent-json (stats/share valid-serialized-json?)+ :percent-url (stats/share u/url?)+ :percent-email (stats/share u/email?)+ :percent-state (stats/share u/state?)+ :percent-blank (stats/share str/blank?)+ :average-length ((map count) stats/mean)+ :min-length ((map count) stats/min)+ :max-length ((map count) stats/max)+ :mode-stats mode-stats})+ (fn [{:keys [mode-stats] :as fused}]+ (-> fused+ (dissoc :mode-stats)+ (assoc :mode-fraction (:mode-fraction mode-stats)+ :top-3-fraction (:top-3-fraction mode-stats)))))))(defn fingerprint-fields"Return a transducer for fingerprinting a resultset with fields `fields`."diff --git a/src/metabase/interestingness/chart.clj b/src/metabase/interestingness/chart.cljnew file mode 100644index 00000000000..f74869f4792--- /dev/null+++ b/src/metabase/interestingness/chart.clj@@ -0,0 +1,169 @@+(ns metabase.interestingness.chart+ "Chart-level interestingness scoring.++ Operates on the same `chart-config` shape that+ [[metabase.interestingness.chart.stats/compute-chart-stats]] consumes —+ `{:series {name -> {:x :y :x_values :y_values ...}} :display_type ...}` —+ and returns a scalar score in `[0.0, 1.0]` describing how likely the chart+ is to surface a useful insight.++ The score blends three signal groups:+ - **non-degeneracy**: hard gate against flat measures and single-value+ dimensions (a chart no aggregation can rescue)+ - **statistical signal**: presence of trend, outliers, correlations, peaks+ in the distribution — read from the stats map+ - **structure**: reasonable category count, non-trivial data-point count++ Callers who already compute stats can pass them via the 2-arity form to+ avoid recomputation."+ (:require+ [metabase.interestingness.chart.stats :as chart.stats]))++(set! *warn-on-reflection* true)++(defn- variance+ [xs]+ (let [n (count xs)]+ (when (pos? n)+ (let [mean (/ (double (reduce + 0.0 xs)) n)]+ (/ (reduce + 0.0 (map #(let [d (- % mean)] (* d d)) xs)) n)))))++(defn- flat?+ "True if the series' y_values have effectively zero spread."+ [{:keys [y_values], :as _series}]+ (let [v (variance y_values)]+ (or (nil? v) (< v 1e-12))))++(defn- single-valued-x?+ [{:keys [x_values]}]+ (<= (count (distinct x_values)) 1))++(defn- non-degeneracy-score+ "0 if every series is flat or has a single x; softer penalty if any one is.+ 1 when all series look lively."+ [{:keys [series], :as _chart-config}]+ (if (empty? series)+ 0.0+ (let [configs (vals series)+ all-flat? (every? flat? configs)+ any-flat? (some flat? configs)+ all-single? (every? single-valued-x? configs)+ any-single? (some single-valued-x? configs)]+ (cond+ (or all-flat? all-single?) 0.0+ (or any-flat? any-single?) 0.4+ :else 1.0))))++(defmulti ^:private signal-score+ "Read statistical signals from a computed stats map. Returns 0..1."+ {:arglists '([stats])}+ :chart-type)++(defn- trend-interesting?+ [{:keys [direction overall-change-pct]}]+ (or (contains? #{:strongly-increasing :strongly-decreasing} direction)+ (and (contains? #{:increasing :decreasing} direction)+ (some-> overall-change-pct double abs (>= 5.0)))))++(defmethod signal-score :time-series+ [{:keys [series correlations], :as _stats}]+ (if (empty? series)+ 0.0+ (let [per-series (for [[_ s] series]+ (let [has-trend (some-> (:trend s) trend-interesting?)+ has-outlier (seq (:outliers s))+ has-change (seq (:significant-changes s))+ volatile? (contains? #{:high :extreme}+ (some-> (:volatility s) :level))]+ (cond-> 0.0+ has-trend (+ 0.4)+ has-outlier (+ 0.25)+ has-change (+ 0.2)+ volatile? (+ 0.15))))+ base (/ (reduce + 0.0 per-series) (count per-series))+ corr-bonus (if (some #(= :strong (:strength %)) correlations) 0.15 0.0)]+ (min 1.0 (+ base corr-bonus)))))++(defmethod signal-score :categorical+ [{:keys [series]}]+ (if (empty? series)+ 0.0+ (let [scores (for [[_ s] series]+ (let [cc (:category-count s 0)+ has-outlier (seq (:outliers s))+ range-ratio (when-let [{:keys [min max]} (:summary s)]+ (when (and min max (pos? (Math/abs (double max))))+ (/ (- (double max) (double min))+ (Math/abs (double max)))))]+ (cond-> 0.0+ (<= 3 cc 50) (+ 0.4)+ (< 50 cc 200) (+ 0.2)+ has-outlier (+ 0.2)+ (and range-ratio (> range-ratio 0.25)) (+ 0.25))))]+ (min 1.0 (/ (reduce + 0.0 scores) (count scores))))))++(defmethod signal-score :scatter+ [{:keys [series]}]+ (if (empty? series)+ 0.0+ (let [scores (for [[_ s] series]+ (let [coef (some-> s :correlation :coefficient double abs)+ r2 (some-> s :regression :r-squared double)+ has-outlier (seq (:outliers s))]+ (cond-> 0.0+ (and coef (>= coef 0.7)) (+ 0.5)+ (and coef (< 0.4 coef 0.7)) (+ 0.3)+ (and r2 (>= r2 0.3)) (+ 0.2)+ has-outlier (+ 0.15))))]+ (min 1.0 (/ (reduce + 0.0 scores) (count scores))))))++(defmethod signal-score :histogram+ [{:keys [series]}]+ (if (empty? series)+ 0.0+ (let [scores (for [[_ s] series]+ (let [{:keys [peak-count concentration-top3]} (:structure s)+ skew (some-> s :distribution :weighted-skewness double abs)]+ (cond-> 0.0+ (and peak-count (>= peak-count 2)) (+ 0.35)+ (and concentration-top3 (> concentration-top3 0.5)) (+ 0.25)+ (and skew (> skew 1.0)) (+ 0.3))))]+ (min 1.0 (/ (reduce + 0.0 scores) (count scores))))))++(defmethod signal-score :default+ [_]+ 0.2)++(defn- structure-score+ "Reward reasonable data shape (enough points, not too many empty series)."+ [chart-config]+ (let [configs (vals (:series chart-config))+ point-counts (map #(count (:y_values %)) configs)+ total (reduce + 0 point-counts)]+ (cond+ (empty? configs) 0.0+ (< total 3) 0.1+ (< total 10) 0.5+ (< total 10000) 1.0+ :else 0.8)))++(defn chart-interestingness+ "Compute a `[0.0, 1.0]` interestingness score for a chart.++ `chart-config` is the same map shape consumed by+ [[metabase.interestingness.chart.stats/compute-chart-stats]]. Optional+ `stats` lets callers who already computed stats avoid recomputation."+ ([chart-config]+ (chart-interestingness chart-config+ (chart.stats/compute-chart-stats chart-config {})))+ ([chart-config stats]+ (let [nd (non-degeneracy-score chart-config)]+ (if (zero? nd)+ 0.0+ (let [sig (signal-score stats)+ struct (structure-score chart-config)]+ (min 1.0+ (max 0.0+ (+ (* 0.3 nd)+ (* 0.5 sig)+ (* 0.2 struct)))))))))diff --git a/src/metabase/metabot/stats/categorical.clj b/src/metabase/interestingness/chart/categorical.cljsimilarity index 92%rename from src/metabase/metabot/stats/categorical.cljrename to src/metabase/interestingness/chart/categorical.cljindex 1167e21863a..97f426764e4 100644--- a/src/metabase/metabot/stats/categorical.clj+++ b/src/metabase/interestingness/chart/categorical.clj@@ -1,9 +1,9 @@-(ns metabase.metabot.stats.categorical+(ns metabase.interestingness.chart.categorical"Categorical chart statistics."(:require- [metabase.metabot.stats.outliers :as outliers]- [metabase.metabot.stats.types :as stats.types]- [metabase.metabot.stats.util :as stats.u]+ [metabase.interestingness.chart.outliers :as outliers]+ [metabase.interestingness.chart.types :as stats.types]+ [metabase.interestingness.chart.util :as stats.u][metabase.util.malli :as mu]))(set! *warn-on-reflection* true)diff --git a/src/metabase/metabot/stats/histogram.clj b/src/metabase/interestingness/chart/histogram.cljsimilarity index 98%rename from src/metabase/metabot/stats/histogram.cljrename to src/metabase/interestingness/chart/histogram.cljindex 6b1e42ceb2f..888251fec8b 100644--- a/src/metabase/metabot/stats/histogram.clj+++ b/src/metabase/interestingness/chart/histogram.clj@@ -1,8 +1,8 @@-(ns metabase.metabot.stats.histogram+(ns metabase.interestingness.chart.histogram"Histogram chart statistics."(:require- [metabase.metabot.stats.types :as stats.types]- [metabase.metabot.stats.util :as stats.u]+ [metabase.interestingness.chart.types :as stats.types]+ [metabase.interestingness.chart.util :as stats.u][metabase.util.malli :as mu]))(set! *warn-on-reflection* true)diff --git a/src/metabase/metabot/stats/outliers.clj b/src/metabase/interestingness/chart/outliers.cljsimilarity index 97%rename from src/metabase/metabot/stats/outliers.cljrename to src/metabase/interestingness/chart/outliers.cljindex 8953f677dc8..cdd3131b0af 100644--- a/src/metabase/metabot/stats/outliers.clj+++ b/src/metabase/interestingness/chart/outliers.clj@@ -1,11 +1,11 @@-(ns metabase.metabot.stats.outliers+(ns metabase.interestingness.chart.outliers"Modified Z-score based outlier detection for chart analysis.Uses the Modified Z-score method which is more robust than standard Z-scoresbecause it uses median and MAD (Median Absolute Deviation) instead of meanand standard deviation, making it resistant to outliers in the calculation itself."(:require- [metabase.metabot.stats.types :as stats.types]+ [metabase.interestingness.chart.types :as stats.types][metabase.util.malli :as mu][tech.v3.datatype.argops :as argops][tech.v3.datatype.functional :as dfn]))diff --git a/src/metabase/metabot/stats/repr.clj b/src/metabase/interestingness/chart/repr.cljsimilarity index 99%rename from src/metabase/metabot/stats/repr.cljrename to src/metabase/interestingness/chart/repr.cljindex 6c8d0f64d63..42511a15c1b 100644--- a/src/metabase/metabot/stats/repr.clj+++ b/src/metabase/interestingness/chart/repr.clj@@ -1,8 +1,8 @@-(ns metabase.metabot.stats.repr+(ns metabase.interestingness.chart.repr"LLM-friendly text representation generator for chart statistics."(:require[clojure.string :as str]- [metabase.metabot.stats.types :as stats.types]+ [metabase.interestingness.chart.types :as stats.types][metabase.util.malli :as mu])(:import(java.time LocalDate LocalDateTime ZonedDateTime)diff --git a/src/metabase/metabot/stats/scatter.clj b/src/metabase/interestingness/chart/scatter.cljsimilarity index 93%rename from src/metabase/metabot/stats/scatter.cljrename to src/metabase/interestingness/chart/scatter.cljindex 0acff627274..1d3a7a6cb28 100644--- a/src/metabase/metabot/stats/scatter.clj+++ b/src/metabase/interestingness/chart/scatter.clj@@ -1,9 +1,9 @@-(ns metabase.metabot.stats.scatter+(ns metabase.interestingness.chart.scatter"Scatter plot statistics."(:require- [metabase.metabot.stats.outliers :as outliers]- [metabase.metabot.stats.types :as stats.types]- [metabase.metabot.stats.util :as stats.u]+ [metabase.interestingness.chart.outliers :as outliers]+ [metabase.interestingness.chart.types :as stats.types]+ [metabase.interestingness.chart.util :as stats.u][metabase.util.malli :as mu][tech.v3.datatype.functional :as dfn]))diff --git a/src/metabase/metabot/stats/core.clj b/src/metabase/interestingness/chart/stats.cljsimilarity index 95%rename from src/metabase/metabot/stats/core.cljrename to src/metabase/interestingness/chart/stats.cljindex 0360f02c2d7..f9387861a8c 100644--- a/src/metabase/metabot/stats/core.clj+++ b/src/metabase/interestingness/chart/stats.clj@@ -1,11 +1,11 @@-(ns metabase.metabot.stats.core+(ns metabase.interestingness.chart.stats"Chart type detection and statistics routing."(:require- [metabase.metabot.stats.categorical :as categorical]- [metabase.metabot.stats.histogram :as histogram]- [metabase.metabot.stats.scatter :as scatter]- [metabase.metabot.stats.time-series :as time-series]- [metabase.metabot.stats.types :as stats.types]+ [metabase.interestingness.chart.categorical :as categorical]+ [metabase.interestingness.chart.histogram :as histogram]+ [metabase.interestingness.chart.scatter :as scatter]+ [metabase.interestingness.chart.time-series :as time-series]+ [metabase.interestingness.chart.types :as stats.types][metabase.util.malli :as mu]))(set! *warn-on-reflection* true)diff --git a/src/metabase/metabot/stats/time_series.clj b/src/metabase/interestingness/chart/time_series.cljsimilarity index 97%rename from src/metabase/metabot/stats/time_series.cljrename to src/metabase/interestingness/chart/time_series.cljindex b0906e9ab37..63b4ea6a20b 100644--- a/src/metabase/metabot/stats/time_series.clj+++ b/src/metabase/interestingness/chart/time_series.clj@@ -1,13 +1,13 @@-(ns metabase.metabot.stats.time-series+(ns metabase.interestingness.chart.time-series"Time series statistics computation for chart analysis.Provides both basic stats (always computed) and deep stats (optional):- Basic: summary, time range, trend, cumulative detection, outliers- Deep: volatility, patterns, significant changes, correlations"(:require- [metabase.metabot.stats.outliers :as outliers]- [metabase.metabot.stats.types :as stats.types]- [metabase.metabot.stats.util :as stats.u]+ [metabase.interestingness.chart.outliers :as outliers]+ [metabase.interestingness.chart.types :as stats.types]+ [metabase.interestingness.chart.util :as stats.u][metabase.util.malli :as mu][tech.v3.datatype.functional :as dfn]))diff --git a/src/metabase/metabot/stats/types.clj b/src/metabase/interestingness/chart/types.cljsimilarity index 99%rename from src/metabase/metabot/stats/types.cljrename to src/metabase/interestingness/chart/types.cljindex 13b7f88779e..0875b8c276d 100644--- a/src/metabase/metabot/stats/types.clj+++ b/src/metabase/interestingness/chart/types.clj@@ -1,4 +1,4 @@-(ns metabase.metabot.stats.types+(ns metabase.interestingness.chart.types"Malli schemas for chart statistics computation."(:require[metabase.util.malli.registry :as mr]))diff --git a/src/metabase/metabot/stats/util.clj b/src/metabase/interestingness/chart/util.cljsimilarity index 98%rename from src/metabase/metabot/stats/util.cljrename to src/metabase/interestingness/chart/util.cljindex 41d22601820..80c16de3097 100644--- a/src/metabase/metabot/stats/util.clj+++ b/src/metabase/interestingness/chart/util.clj@@ -1,7 +1,7 @@-(ns metabase.metabot.stats.util+(ns metabase.interestingness.chart.util"Shared utilities used across various stats chart types."(:require- [metabase.metabot.stats.types :as stats.types]+ [metabase.interestingness.chart.types :as stats.types][metabase.util.malli :as mu][tech.v3.datatype.functional :as dfn]))diff --git a/src/metabase/interestingness/core.clj b/src/metabase/interestingness/core.cljnew file mode 100644index 00000000000..cf88285b5b1--- /dev/null+++ b/src/metabase/interestingness/core.clj@@ -0,0 +1,27 @@+(ns metabase.interestingness.core+ "Public entry point for the interestingness engine.++ This namespace is a thin facade — it only re-exports the user-facing+ `compute interestingness for X` functions from their role-specific+ implementation namespaces. If you need scorers, weight profiles, or the+ composition machinery, depend on the specific namespace directly:++ - [[metabase.interestingness.chart]] — chart-level scoring + stats helpers+ - [[metabase.interestingness.dimension]] — dimension scorers + canonical weights+ - [[metabase.interestingness.impl]] — shared composition machinery (internal)"+ (:require+ [metabase.interestingness.chart]+ [metabase.interestingness.chart.repr]+ [metabase.interestingness.chart.stats]+ [metabase.interestingness.dimension]+ [potemkin :as p]))++(p/import-vars+ [metabase.interestingness.chart+ chart-interestingness]+ [metabase.interestingness.dimension+ dimension-interestingness]+ [metabase.interestingness.chart.repr+ generate-representation]+ [metabase.interestingness.chart.stats+ compute-chart-stats])diff --git a/src/metabase/interestingness/dimension.clj b/src/metabase/interestingness/dimension.cljnew file mode 100644index 00000000000..59af815b402--- /dev/null+++ b/src/metabase/interestingness/dimension.clj@@ -0,0 +1,195 @@+(ns metabase.interestingness.dimension+ "Dimension-role interestingness scoring."+ (:require+ [metabase.interestingness.impl :as impl]+ [metabase.util.time :as u.time]))++(set! *warn-on-reflection* true)++(defn- bucket-count-score+ "Universal quality curve for bucket/group counts. Sweet spot is 10-25 buckets.+ Returns 0.0 for ≤1 (hard-zero gate for constant fields)."+ [n]+ (cond+ (<= n 1) 0.0+ (<= n 2) 0.15+ (<= n 5) (+ 0.15 (* (/ (- n 2.0) 3.0) 0.25)) ; 2→0.15 .. 5→0.4+ (<= n 10) (+ 0.4 (* (/ (- n 5.0) 5.0) 0.4)) ; 5→0.4 .. 10→0.8+ (<= n 25) 1.0 ; sweet spot+ (<= n 50) (- 1.0 (* (/ (- n 25.0) 25.0) 0.3)) ; 25→1.0 .. 50→0.7+ (<= n 100) (- 0.7 (* (/ (- n 50.0) 50.0) 0.4)) ; 50→0.7 .. 100→0.3+ :else (max 0.05 (- 0.3 (* 0.05 (Math/log (/ n 100.0)))))))++(def ^:private temporal-bucket-days+ "Temporal bucketing strategies with approximate days per bucket, coarsest first+ so `reduce` finds the best match efficiently (coarser = fewer buckets, finer+ granularities only win if coarser ones overshoot)."+ [[:year 365.25]+ [:quarter 91.31]+ [:month 30.44]+ [:week 7.0]+ [:day 1.0]+ [:hour (/ 1.0 24)]+ [:minute (/ 1.0 1440)]])++(defn- best-temporal-buckets+ "Given a day span, find the granularity whose bucket count scores best."+ [day-span]+ (if (<= day-span 0)+ {:granularity :day :count 1 :score 0.0}+ (reduce (fn [best [gran days-per]]+ (let [n (/ day-span days-per)+ s (bucket-count-score n)]+ (if (> s (:score best)) {:granularity gran :count n :score s} best)))+ {:granularity :year :count (/ day-span 365.25) :score 0.0}+ temporal-bucket-days)))++(defn cardinality+ "Score based on best achievable bucket/group count for visualization.+ - Temporal fields: evaluates granularities (day/week/month/quarter/year) and picks+ the one that lands closest to the 10-25 bucket sweet spot.+ - Numeric fields with high distinct count and a range: auto-binnable, scores high.+ - All others (categorical, low-cardinality numeric): scores by raw distinct count.+ Returns 0.0 (hard-zero gate) for constant fields (≤1 distinct value)."+ [field]+ (let [distinct-count (get-in field [:fingerprint :global :distinct-count])+ temporal-fp (get-in field [:fingerprint :type :type/DateTime])+ numeric-fp (get-in field [:fingerprint :type :type/Number])]+ (cond+ ;; No fingerprint data at all+ (nil? distinct-count)+ {:score 0.5 :reason "no cardinality data"}++ ;; Constant field — hard-zero gate+ (<= distinct-count 1)+ {:score 0.0 :reason "constant field"}++ ;; Temporal with range data → pick best granularity+ (and temporal-fp (:earliest temporal-fp) (:latest temporal-fp))+ (let [{:keys [earliest latest]} temporal-fp+ start (u.time/coerce-to-timestamp earliest)+ end (u.time/coerce-to-timestamp latest)]+ (if (and start end)+ (let [days (u.time/day-diff start end)+ {:keys [granularity count score]} (best-temporal-buckets days)]+ {:score score+ :reason (str (long days) "-day range, ~" (long count) " " (name granularity) " buckets")})+ {:score 0.5 :reason "unparseable temporal bounds"}))++ ;; Numeric with high cardinality and a nonzero range → auto-binning will produce good buckets+ (and numeric-fp (> distinct-count 50)+ (some? (:min numeric-fp)) (some? (:max numeric-fp))+ (not= (:min numeric-fp) (:max numeric-fp)))+ {:score 0.9 :reason (str distinct-count " distinct values, auto-binnable")}++ ;; Everything else: score by raw distinct count (categoricals, low-cardinality numerics, etc.)+ :else+ {:score (bucket-count-score distinct-count)+ :reason (str distinct-count " distinct values")})))++(def ^:private dimension-bonus-types+ "Semantic types that make especially interesting dimensions."+ [:type/Address+ :type/Birthdate+ :type/CancelationTemporal+ :type/Company+ :type/CreationTimestamp+ :type/Currency+ :type/JoinTemporal+ :type/Owner+ :type/Quantity+ :type/Score+ :type/Share+ :type/Subscription+ :type/Title])++(defn type-bonus+ "Boost fields with semantic types that tend to produce interesting explorations."+ [field]+ (let [semantic-type (:semantic-type field)]+ (if (some #(isa? semantic-type %) dimension-bonus-types)+ {:score 1.0 :reason "special type"}+ {:score 0.5 :reason "no type bonus"})))++(defn temporal-range+ "Score temporal fields by their date range span. Wider ranges enable richer time-series exploration."+ [field]+ (let [temporal-fp (get-in field [:fingerprint :type :type/DateTime])]+ (if (nil? temporal-fp)+ {:score 0.5 :reason "not a temporal field"}+ (let [{:keys [earliest latest]} temporal-fp]+ (if (or (nil? earliest) (nil? latest))+ {:score 0.5 :reason "missing temporal bounds"}+ (let [start (u.time/coerce-to-timestamp earliest)+ end (u.time/coerce-to-timestamp latest)]+ (if (or (nil? start) (nil? end))+ {:score 0.5 :reason "unparseable temporal bounds"}+ (let [days (u.time/day-diff start end)]+ (cond+ (<= days 0) {:score 0.1 :reason "single-point temporal range"}+ (< days 7) {:score 0.4 :reason (str days " day range")}+ (< days 30) {:score 0.6 :reason (str days " day range")}+ (< days 365) {:score 0.8 :reason (str days " day range")}+ :else {:score 0.95 :reason (str days " day range")})))))))))++(defn text-structure+ "Penalize text fields that are structured data (JSON, URLs, email, state codes), mostly+ blank, or free-form long text. These make poor breakout candidates.++ Email / state percentage checks are a safety net for fields the classifier missed:+ normally they get `:type/Email` / `:type/State` semantic types which are handled by+ `type-bonus`, but for borderline or misclassified fields the raw percentages still+ catch them here."+ [field]+ (let [text-fp (get-in field [:fingerprint :type :type/Text])]+ (if (nil? text-fp)+ {:score 0.5 :reason "not a text field"}+ (let [{:keys [percent-json percent-url percent-email percent-state average-length percent-blank]} text-fp]+ (cond+ (and (some? percent-blank) (> percent-blank 0.8))+ {:score 0.1 :reason (str (long (* 100 percent-blank)) "% blank/empty")}++ (and (some? percent-json) (> percent-json 0.9))+ {:score 0.1 :reason (str (long (* 100 percent-json)) "% JSON content")}++ (and (some? percent-url) (> percent-url 0.9))+ {:score 0.15 :reason (str (long (* 100 percent-url)) "% URL content")}++ (and (some? percent-email) (> percent-email 0.9))+ {:score 0.2 :reason (str (long (* 100 percent-email)) "% email content (likely PII, poor breakout)")}++ (and (some? percent-state) (> percent-state 0.9))+ {:score 0.4 :reason (str (long (* 100 percent-state)) "% state codes (a map visualization fits better)")}++ (and (some? average-length) (> average-length 100))+ {:score 0.05 :reason (str "avg length " (long average-length) " chars")}++ (and (some? average-length) (> average-length 50))+ {:score 0.2 :reason (str "avg length " (long average-length) " chars")}++ :else+ {:score 0.8 :reason "short structured text"})))))++;;; -------------------------------------------------- Weight profiles --------------------------------------------------++(def canonical-dimension-weights+ "Canonical weight profile for scoring a field as a *dimension* (breakout column).+ Persisted as `dimension_interestingness` on metabase_field. Rewards structural+ cleanliness, good bucket counts, category/temporal types, and balanced distributions."+ {impl/type-penalty 0.30+ cardinality 0.20+ impl/nullness 0.10+ type-bonus 0.10+ temporal-range 0.05+ text-structure 0.05+ impl/distribution-shape 0.15+ impl/numeric-variance 0.05})++(defn dimension-interestingness+ "Return the canonical `dimension_interestingness` score for `field`.++ Accepts either a raw DB-style field map with snake_case keys or a normalized+ field map with kebab-case keys. Returns a double in [0.0, 1.0]."+ [field]+ (:score (impl/score-field canonical-dimension-weights+ (impl/normalize-field field))))diff --git a/src/metabase/interestingness/impl.clj b/src/metabase/interestingness/impl.cljnew file mode 100644index 00000000000..aa616c229a4--- /dev/null+++ b/src/metabase/interestingness/impl.clj@@ -0,0 +1,232 @@+(ns metabase.interestingness.impl+ "Shared composition machinery and role-independent scorers for the interestingness+ engine.++ Scorer contract: `(fn [field]) -> {:score double, :reason string}`+ - field: map of field/dimension metadata (kebab-case keys)+ - score: double in [0.0, 1.0] where 0.0 = uninteresting, 1.0 = very interesting+ - reason: short human-readable explanation++ Score semantics:+ 0.0 Hard exclude (e.g. PK, hidden field)+ 0.01-0.29 Very low value, likely noise+ 0.30-0.49 Below average+ 0.50 Neutral / unknown (missing metadata)+ 0.51-0.74 Decent, typical useful field+ 0.75-1.0 High value for exploration"+ (:require+ [clojure.set :as set]))++(set! *warn-on-reflection* true)++;;; -------------------------------------------------- Shared scorers --------------------------------------------------++(defn type-penalty+ "Hard-zero fields whose data values carry no exploratory meaning regardless of the+ role they're being scored for (dimension or measure):+ - `:type/PK`: opaque row identifiers+ - `:type/Collection` / `:type/Structured`: structured blobs (JSON, XML, arrays,+ dictionaries, text-stored serialized JSON) — not groupable or aggregatable+ - `:type/UpdatedTimestamp` / `:type/DeletionTimestamp`: audit fields that describe+ the record, not the entity"+ [field]+ (let [semantic-type (:semantic-type field)]+ (cond+ (nil? semantic-type) {:score 1.0 :reason "no semantic type"}+ (isa? semantic-type :type/PK) {:score 0.0 :reason "primary key"}+ (isa? semantic-type :type/Collection) {:score 0.0 :reason "structured blob"}+ (isa? semantic-type :type/Structured) {:score 0.0 :reason "structured blob"}+ (isa? semantic-type :type/UpdatedTimestamp) {:score 0.0 :reason "updated timestamp"}+ (isa? semantic-type :type/DeletionTimestamp) {:score 0.0 :reason "deletion timestamp"}+ :else {:score 1.0 :reason "no type penalty"})))++(defn nullness+ "Linear penalty based on null percentage. Mostly-null fields are noise whether being+ used as a dimension (empty bars) or a measure (nothing to aggregate)."+ [field]+ (if-let [nil-pct (get-in field [:fingerprint :global :nil%])]+ {:score (- 1.0 nil-pct)+ :reason (if (> nil-pct 0.95)+ "mostly null"+ (str (long (* 100 nil-pct)) "% null"))}+ {:score 0.5 :reason "no null data"}))++(defn numeric-variance+ "Score numeric fields by their statistical spread. Zero-variance fields are+ uninteresting regardless of role — nothing to show as a dim (one bucket) or+ as a measure (SUM of a constant)."+ [field]+ (let [num-fp (get-in field [:fingerprint :type :type/Number])]+ (if (nil? num-fp)+ {:score 0.5 :reason "not a numeric field"}+ (let [{:keys [sd q1 q3 avg], mn :min, mx :max} num-fp]+ (cond+ (and (some? sd) (zero? sd))+ {:score 0.0 :reason "zero variance"}++ (and (some? q1) (some? q3) (== q1 q3))+ {:score 0.1 :reason "no interquartile spread"}++ (and (some? sd) (some? avg) (not (zero? avg)))+ (let [cv (abs (/ sd avg))+ score (min 1.0 (max 0.2 (* 0.5 (+ 1.0 (Math/log (+ 1.0 cv))))))]+ {:score score :reason (str "coefficient of variation: " (double cv))})++ (and (some? q1) (some? q3) (some? mn) (some? mx) (not (== mn mx)))+ (let [iqr (- q3 q1)+ rng (- mx mn)+ ratio (/ iqr rng)+ score (min 1.0 (max 0.2 (+ 0.3 (* 0.7 ratio))))]+ {:score score :reason (str "IQR ratio: " (double ratio))})++ :else+ {:score 0.5 :reason "insufficient numeric stats"})))))++(defn- skewness-score+ [skewness]+ (let [abs-sk (abs (double skewness))]+ (cond+ (< abs-sk 0.5) {:score 1.0 :reason (str "symmetric distribution (skewness " (format "%.2f" (double skewness)) ")")}+ (< abs-sk 1.0) {:score 0.85 :reason (str "slightly skewed (skewness " (format "%.2f" (double skewness)) ")")}+ (< abs-sk 2.0) {:score 0.6 :reason (str "moderately skewed (skewness " (format "%.2f" (double skewness)) ")")}+ (< abs-sk 4.0) {:score 0.3 :reason (str "heavily skewed (skewness " (format "%.2f" (double skewness)) ")")}+ :else {:score 0.15 :reason (str "extremely skewed (skewness " (format "%.2f" (double skewness)) ")")})))++(defn- kurtosis-score+ [kurtosis]+ (let [k (double kurtosis)]+ (cond+ (< -1.0 k 3.0) {:score 1.0 :reason (str "normal-ish tails (kurtosis " (format "%.2f" k) ")")}+ (< -2.0 k 7.0) {:score 0.7 :reason (str "moderately heavy tails (kurtosis " (format "%.2f" k) ")")}+ (< -3.0 k 15.0) {:score 0.4 :reason (str "heavy tails / outlier-dominated (kurtosis " (format "%.2f" k) ")")}+ :else {:score 0.15 :reason (str "extreme tail weight (kurtosis " (format "%.2f" k) ")")})))++(defn- mode-dominance-score+ [mode-fraction]+ (cond+ (>= mode-fraction 0.95) {:score 0.05 :reason (str (long (* 100 mode-fraction)) "% single value (near-constant)")}+ (>= mode-fraction 0.80) {:score 0.2 :reason (str (long (* 100 mode-fraction)) "% single value (heavily dominated)")}+ (>= mode-fraction 0.60) {:score 0.5 :reason (str (long (* 100 mode-fraction)) "% single value")}+ (>= mode-fraction 0.40) {:score 0.75 :reason (str (long (* 100 mode-fraction)) "% single value")}+ :else {:score 1.0 :reason (str (long (* 100 mode-fraction)) "% single value (well-distributed)")}))++(defn- top-3-concentration-score+ [top-3-fraction]+ (cond+ (>= top-3-fraction 0.99) {:score 0.1 :reason (str (long (* 100 top-3-fraction)) "% in top 3 (effectively 3 values)")}+ (>= top-3-fraction 0.95) {:score 0.3 :reason (str (long (* 100 top-3-fraction)) "% in top 3")}+ :else {:score 1.0 :reason (str (long (* 100 top-3-fraction)) "% in top 3")}))++(defn- zero-dominance-score+ [zero-fraction]+ (cond+ (>= zero-fraction 0.95) {:score 0.05 :reason (str (long (* 100 zero-fraction)) "% zeros (sparse/empty)")}+ (>= zero-fraction 0.80) {:score 0.2 :reason (str (long (* 100 zero-fraction)) "% zeros")}+ (>= zero-fraction 0.60) {:score 0.6 :reason (str (long (* 100 zero-fraction)) "% zeros")}+ :else {:score 1.0 :reason (str (long (* 100 zero-fraction)) "% zeros")}))++(defn distribution-shape+ "Score fields by the shape of their value distribution. Penalizes fields where:+ - A single value dominates (high mode-fraction): near-constant, poor for visualization+ - Top-3 values cover nearly everything (few real categories)+ - Zero values dominate (numeric counters / sparse data)+ - Distribution is heavily skewed or has extreme tail weight (kurtosis)++ Works on numeric, text, and temporal fields, and is role-neutral: a flat/skewed+ distribution is boring both as a breakout and as an aggregation target. Returns 0.5+ (neutral) when no distribution data is available. Signals are combined worst-of-N:+ any single strong boringness signal dominates the score."+ [field]+ (let [num-fp (get-in field [:fingerprint :type :type/Number])+ text-fp (get-in field [:fingerprint :type :type/Text])+ date-fp (get-in field [:fingerprint :type :type/DateTime])+ mode-frac (or (:mode-fraction num-fp) (:mode-fraction text-fp) (:mode-fraction date-fp))+ top3 (or (:top-3-fraction num-fp) (:top-3-fraction text-fp) (:top-3-fraction date-fp))+ skewness (or (:skewness num-fp) (:skewness date-fp))+ kurtosis (:excess-kurtosis num-fp)+ zero-frac (:zero-fraction num-fp)+ sub-scores (cond-> []+ (some? mode-frac) (conj (mode-dominance-score mode-frac))+ (some? top3) (conj (top-3-concentration-score top3))+ (some? zero-frac) (conj (zero-dominance-score zero-frac))+ (some? skewness) (conj (skewness-score skewness))+ (some? kurtosis) (conj (kurtosis-score kurtosis)))]+ (if (empty? sub-scores)+ {:score 0.5 :reason "no distribution data"}+ (let [worst (apply min-key :score sub-scores)]+ {:score (:score worst)+ :reason (:reason worst)}))))++;;; -------------------------------------------------- Composition --------------------------------------------------++(defn score-field+ "Score a single field using weighted scorers. Returns:+ {:score double ;; weighted average in [0.0, 1.0]+ :scores {scorer-fn {:score double, :reason string}} ;; per-scorer breakdown+ :field field} ;; the input field, passed through++ If any scorer with nonzero weight returns exactly 0.0, the final score is clamped+ to at most 0.1. This lets hard signals act as effective gates regardless of what+ other scorers return."+ [scorer-weight-map field]+ (let [total-weight (reduce + 0.0 (vals scorer-weight-map))+ results (reduce-kv+ (fn [acc scorer weight]+ (let [{:keys [score] :as result} (scorer field)]+ (-> acc+ (update :weighted-sum + (* weight score))+ (update :has-hard-zero? #(or % (and (pos? weight) (zero? score))))+ (assoc-in [:scores scorer] result))))+ {:weighted-sum 0.0 :scores {} :has-hard-zero? false}+ scorer-weight-map)+ raw-score (if (pos? total-weight)+ (/ (:weighted-sum results) total-weight)+ 0.5)+ final-score (if (:has-hard-zero? results)+ (min raw-score 0.1)+ raw-score)]+ {:score final-score+ :scores (:scores results)+ :field field}))++(defn compose+ "Combine multiple scorers with weights into a single scorer function.+ Returns `(fn [field]) -> {:score double, :scores map, :field map}`."+ [scorer-weight-map]+ (fn [field] (score-field scorer-weight-map field)))++(defn apply-cutoff+ "Filter a sequence of scored field results, keeping only those at or above `threshold`."+ [threshold scored-fields]+ (filter #(>= (:score %) threshold) scored-fields))++(defn score-and-filter+ "Score all fields with the given scorer-weight-map and return only those at or above `cutoff`.+ Results are sorted by score descending."+ [scorer-weight-map fields cutoff]+ (->> fields+ (map #(score-field scorer-weight-map %))+ (apply-cutoff cutoff)+ (sort-by :score >)))++;;; -------------------------------------------------- Field Normalization --------------------------------------------------++(def ^:private snake->kebab-keys+ "Keys that need conversion from snake_case (raw DB Field maps) to kebab-case (scorer input)."+ {:semantic_type :semantic-type+ :base_type :base-type+ :effective_type :effective-type+ :visibility_type :visibility-type})++(defn normalize-field+ "Convert a snake_case field map (as used in x-rays/raw DB) to the kebab-case shape scorers expect.+ Keys not in the rename map are passed through unchanged (e.g., :fingerprint)."+ [field]+ (set/rename-keys field snake->kebab-keys))++(defn score-raw-field+ "Score a snake_case field map by normalizing it first. Returns the same shape as `score-field`+ but with the original (unnormalized) field in `:field`."+ [scorer-weight-map field]+ (let [result (score-field scorer-weight-map (normalize-field field))]+ (assoc result :field field)))diff --git a/src/metabase/lib/schema/metadata/fingerprint.cljc b/src/metabase/lib/schema/metadata/fingerprint.cljcindex 7f28f2a49e7..63b7c6ca851 100644--- a/src/metabase/lib/schema/metadata/fingerprint.cljc+++ b/src/metabase/lib/schema/metadata/fingerprint.cljc@@ -22,12 +22,17 @@"Schema for fingerprint information for Fields deriving from `:type/Number`."[:map{:decode/normalize lib.schema.common/normalize-map-no-kebab-case}- [:min {:optional true} [:maybe number?]]- [:max {:optional true} [:maybe number?]]- [:avg {:optional true} [:maybe number?]]- [:q1 {:optional true} [:maybe number?]]- [:q3 {:optional true} [:maybe number?]]- [:sd {:optional true} [:maybe number?]]])+ [:min {:optional true} [:maybe number?]]+ [:max {:optional true} [:maybe number?]]+ [:avg {:optional true} [:maybe number?]]+ [:q1 {:optional true} [:maybe number?]]+ [:q3 {:optional true} [:maybe number?]]+ [:sd {:optional true} [:maybe number?]]+ [:skewness {:optional true} [:maybe number?]]+ [:excess-kurtosis {:optional true} [:maybe number?]]+ [:mode-fraction {:optional true} [:maybe [:ref ::percent]]]+ [:top-3-fraction {:optional true} [:maybe [:ref ::percent]]]+ [:zero-fraction {:optional true} [:maybe [:ref ::percent]]]])(mr/def ::fingerprint.text"Schema for fingerprint information for Fields deriving from `:type/Text`."@@ -37,14 +42,24 @@[:percent-url {:optional true} [:maybe [:ref ::percent]]][:percent-email {:optional true} [:maybe [:ref ::percent]]][:percent-state {:optional true} [:maybe [:ref ::percent]]]- [:average-length {:optional true} [:maybe number?]]])+ [:percent-blank {:optional true} [:maybe [:ref ::percent]]]+ [:average-length {:optional true} [:maybe number?]]+ [:min-length {:optional true} [:maybe number?]]+ [:max-length {:optional true} [:maybe number?]]+ [:mode-fraction {:optional true} [:maybe [:ref ::percent]]]+ [:top-3-fraction {:optional true} [:maybe [:ref ::percent]]]])(mr/def ::fingerprint.temporal"Schema for fingerprint information for Fields deriving from `:type/Temporal`."[:map{:decode/normalize lib.schema.common/normalize-map-no-kebab-case}- [:earliest {:optional true} [:maybe :string]]- [:latest {:optional true} [:maybe :string]]])+ [:earliest {:optional true} [:maybe :string]]+ [:latest {:optional true} [:maybe :string]]+ [:skewness {:optional true} [:maybe number?]]+ [:mode-fraction {:optional true} [:maybe [:ref ::percent]]]+ [:top-3-fraction {:optional true} [:maybe [:ref ::percent]]]+ [:weekday-distribution {:optional true} [:maybe [:sequential [:ref ::percent]]]]+ [:hour-distribution {:optional true} [:maybe [:sequential [:ref ::percent]]]]])(mr/def ::fingerprint.type-specific"Schema for type-specific fingerprint information."diff --git a/src/metabase/metabot/tools/analyze_chart.clj b/src/metabase/metabot/tools/analyze_chart.cljindex 0312d62693d..d406a1c3fef 100644--- a/src/metabase/metabot/tools/analyze_chart.clj+++ b/src/metabase/metabot/tools/analyze_chart.clj@@ -5,9 +5,8 @@series data in chart_configs, which is seeded into agent memory duringinitialization. This avoids re-executing queries."(:require+ [metabase.interestingness.core :as interestingness][metabase.metabot.scope :as scope]- [metabase.metabot.stats.core :as stats.core]- [metabase.metabot.stats.repr :as stats.repr][metabase.metabot.tools.shared :as shared][metabase.util.log :as log][metabase.util.malli :as mu]@@ -68,12 +67,12 @@ Do not use headers (##). Do not list statistics. Do not analyze series separatel(resource/stack-resource-context(let [{:keys [timeline_events title display_type]} chart-configopts {:deep? (if (nil? deep) true (boolean deep))}- stats (stats.core/compute-chart-stats chart-config opts)+ stats (interestingness/compute-chart-stats chart-config opts)context {:title title:display-type display_type:stats stats:timeline-events timeline_events}- representation (stats.repr/generate-representation context)]+ representation (interestingness/generate-representation context)]{:output (str representation "\n\n---\n\n" interpretation-guidance)})){:output (str "Chart config not found: " chart_config_id". Available chart configs can be found in the viewing context.")})diff --git a/src/metabase/sync/analyze.clj b/src/metabase/sync/analyze.cljindex 9b69cce14c0..9aa73c07eb7 100644--- a/src/metabase/sync/analyze.clj+++ b/src/metabase/sync/analyze.clj@@ -6,6 +6,7 @@(:require[metabase.sync.analyze.classify :as classify][metabase.sync.analyze.fingerprint :as sync.fingerprint]+ [metabase.sync.analyze.interestingness :as sync.interestingness][metabase.sync.interface :as i][metabase.sync.util :as sync-util][metabase.util :as u]@@ -75,6 +76,7 @@(sync.fingerprint/fingerprint-table! table)(classify/classify-fields! table)(classify/classify-table! table)+ (sync.interestingness/score-fields! table)(update-fields-last-analyzed! table))(defn- maybe-log-progress [progress-bar-fn]@@ -95,6 +97,10 @@(format "Total number of tables classified %d, %d updated"total-tables tables-classified))+(defn- interestingness-summary [{:keys [fields-scored fields-failed]}]+ (format "Interestingness scored %d fields, %d failed"+ fields-scored fields-failed))+(defn- make-analyze-steps [log-fn][(sync-util/create-sync-step "fingerprint-fields"#(sync.fingerprint/fingerprint-fields-for-db! % log-fn)@@ -104,7 +110,10 @@classify-fields-summary)(sync-util/create-sync-step "classify-tables"#(classify/classify-tables-for-db! % log-fn)- classify-tables-summary)])+ classify-tables-summary)+ (sync-util/create-sync-step "score-interestingness"+ #(sync.interestingness/score-fields-for-db! % log-fn)+ interestingness-summary)])(mu/defn analyze-db!"Perform in-depth analysis on the data for all Tables in a given `database`. This is dependent on what each database@@ -112,7 +121,7 @@`:last_analyzed` value for each affected Field."[database :- i/DatabaseInstance](sync-util/sync-operation :analyze database (format "Analyze data for %s" (sync-util/name-for-logging database))- (sync-util/with-emoji-progress-bar [emoji-progress-bar (inc (* 3 (sync-util/sync-tables-count database)))]+ (sync-util/with-emoji-progress-bar [emoji-progress-bar (inc (* 4 (sync-util/sync-tables-count database)))](u/prog1 (sync-util/run-sync-operation "analyze" database (make-analyze-steps (maybe-log-progress emoji-progress-bar)))(update-fields-last-analyzed-for-db! database)))))diff --git a/src/metabase/sync/analyze/interestingness.clj b/src/metabase/sync/analyze/interestingness.cljnew file mode 100644index 00000000000..18d3c391215--- /dev/null+++ b/src/metabase/sync/analyze/interestingness.clj@@ -0,0 +1,63 @@+(ns metabase.sync.analyze.interestingness+ "Analysis sub-step that computes a canonical dimension-interestingness score for each+ field and persists it on `metabase_field.dimension_interestingness`.++ Runs after fingerprinting and classification so that scorers have both the statistical+ fingerprint and the inferred semantic type available. Scores are recomputed whenever a+ field is re-fingerprinted; there is no separate version tracking."+ (:require+ [metabase.interestingness.core :as interestingness]+ [metabase.sync.analyze.fingerprint :as sync.fingerprint]+ [metabase.sync.interface :as i]+ [metabase.sync.util :as sync-util]+ [metabase.util :as u]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(mu/defn- score-and-save!+ "Score a single field's dimension role and persist the composite score."+ [field :- i/FieldInstance]+ (sync-util/with-error-handling (format "Error scoring interestingness for %s" (sync-util/name-for-logging field))+ (let [dim-score (interestingness/dimension-interestingness field)]+ (t2/update! :model/Field (u/the-id field)+ {:dimension_interestingness dim-score}))))++(mu/defn- fields-to-score :- [:maybe [:sequential i/FieldInstance]]+ "Return Fields in `table` with fresh fingerprints that haven't completed analysis yet."+ [table :- i/TableInstance]+ (seq (apply t2/select :model/Field+ :table_id (u/the-id table)+ :active true+ :visibility_type [:not-in ["sensitive" "retired"]]+ (reduce concat [] (sync.fingerprint/incomplete-analysis-kvs)))))++(mu/defn score-fields!+ "Score interestingness for all qualifying Fields in `table`."+ [table :- i/TableInstance]+ (if-let [fields (fields-to-score table)]+ (do+ (log/debugf "Scoring interestingness for %d fields in %s" (count fields) (sync-util/name-for-logging table))+ (reduce (fn [stats field]+ (let [result (score-and-save! field)]+ (if (instance? Exception result)+ (update stats :fields-failed inc)+ (update stats :fields-scored inc))))+ {:fields-scored 0 :fields-failed 0}+ fields))+ {:fields-scored 0 :fields-failed 0}))++(mu/defn score-fields-for-db!+ "Score interestingness for all qualifying Fields in `database`."+ [database :- i/DatabaseInstance+ log-progress-fn]+ (let [tables (sync-util/reducible-sync-tables database)]+ (transduce (map (fn [table]+ (let [result (score-fields! table)]+ (log-progress-fn "score-interestingness" table)+ result)))+ (partial merge-with +)+ {:fields-scored 0 :fields-failed 0}+ tables)))diff --git a/src/metabase/sync/interface.clj b/src/metabase/sync/interface.cljindex 2a88fa72a95..ffbb748001a 100644--- a/src/metabase/sync/interface.clj+++ b/src/metabase/sync/interface.clj@@ -192,7 +192,8 @@2 #{:type/Number}3 #{:type/DateTime}4 #{:type/*}- 5 #{:type/Text}})+ 5 #{:type/Text}+ 6 #{:type/Number :type/Text :type/DateTime}})(def ^:dynamic ^Long *latest-fingerprint-version*"The newest (highest-numbered) version of our Field fingerprints."diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.cljindex e6240724c44..4e6d0390044 100644--- a/src/metabase/sync/util.clj+++ b/src/metabase/sync/util.clj@@ -319,7 +319,9 @@"😎"]) ; smiling face with sunglasses(defn- percent-done->emoji [percent-done]- (progress-emoji (int (math/round (* percent-done (dec (count progress-emoji)))))))+ (let [last-idx (dec (count progress-emoji))+ idx (int (math/round (* percent-done last-idx)))]+ (progress-emoji (max 0 (min last-idx idx)))))(defn emoji-progress-bar"Create a string that shows progress for something, e.g. a database sync process.diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.cljindex 99e89050417..834ae746098 100644--- a/src/metabase/warehouse_schema/models/field.clj+++ b/src/metabase/warehouse_schema/models/field.clj@@ -460,7 +460,7 @@:description :display_name :effective_type :has_field_values :is_defective_duplicate:json_unfolding :name :nfc_path :points_of_interest :position :preview_display :semantic_type :settings:unique_field_helper :visibility_type]- :skip [:fingerprint :fingerprint_version :last_analyzed]+ :skip [:dimension_interestingness :fingerprint :fingerprint_version :last_analyzed]:transform {:created_at (serdes/date):table_id (serdes/fk :model/Table):fk_target_field_id (serdes/fk :model/Field)diff --git a/test/metabase/analyze/fingerprint/fingerprinters_test.clj b/test/metabase/analyze/fingerprint/fingerprinters_test.cljindex 4c2bee9cf02..e93e9f21f98 100644--- a/test/metabase/analyze/fingerprint/fingerprinters_test.clj+++ b/test/metabase/analyze/fingerprint/fingerprinters_test.clj@@ -12,6 +12,15 @@(set! *warn-on-reflection* true)+(defn- temporal-core-fields+ "Return the subset of a temporal fingerprint result containing the historically-asserted fields.+ We compare only :earliest / :latest (plus :global) to keep tests stable against new distribution+ stats (skewness, mode-fraction, weekday-distribution, hour-distribution)."+ [result]+ {:global (:global result)+ :type {:type/DateTime (select-keys (get-in result [:type :type/DateTime])+ [:earliest :latest])}})+(deftest fingerprint-temporal-values-test;; we want to test h2 and postgres, because h2 doesn't;; support overriding the timezone for a session / report@@ -24,63 +33,70 @@:nil% 0.5}:type {:type/DateTime {:earliest "2013-01-01":latest "2018-01-01"}}}- (transduce identity- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/DateTime}))- [#t "2013" nil #t "2018" nil nil #t "2015"])))+ (temporal-core-fields+ (transduce identity+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/DateTime}))+ [#t "2013" nil #t "2018" nil nil #t "2015"]))))(testing "handle ChronoLocalDateTime"(is (= {:global {:distinct-count 2:nil% 0.0}:type {:type/DateTime {:earliest "2013-01-01T20:04:00Z":latest "2018-01-01T04:04:00Z"}}}- (transduce identity- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))- [(java.time.LocalDateTime/of 2013 01 01 20 04 0 0)- (java.time.LocalDateTime/of 2018 01 01 04 04 0 0)]))))+ (temporal-core-fields+ (transduce identity+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))+ [(java.time.LocalDateTime/of 2013 01 01 20 04 0 0)+ (java.time.LocalDateTime/of 2018 01 01 04 04 0 0)])))))(testing "handle comparing explicit Instant with ChronoLocalDateTime"(is (= {:global {:distinct-count 2:nil% 0.0}:type {:type/DateTime {:earliest "2007-12-03T10:15:30Z":latest "2018-01-01T04:04:00Z"}}}- (transduce identity- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))- [(java.time.Instant/parse "2007-12-03T10:15:30.00Z")- (java.time.LocalDateTime/of 2018 01 01 04 04 0 0)]))))+ (temporal-core-fields+ (transduce identity+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))+ [(java.time.Instant/parse "2007-12-03T10:15:30.00Z")+ (java.time.LocalDateTime/of 2018 01 01 04 04 0 0)])))))(testing "mixing numbers and strings"(is (= {:global {:distinct-count 2:nil% 0.0}:type {:type/DateTime {:earliest "1970-01-01T00:00:01.234Z":latest "2007-12-03T10:15:30Z"}}}- (transduce identity- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))- ["2007-12-03T10:15:30.00Z" 1234]))))+ (temporal-core-fields+ (transduce identity+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))+ ["2007-12-03T10:15:30.00Z" 1234])))))(testing "nil temporal values"(is (= {:global {:distinct-count 1:nil% 1.0}:type {:type/DateTime {:earliest nil:latest nil}}}- (transduce identity- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/DateTime}))- (repeat 10 nil)))))+ (temporal-core-fields+ (transduce identity+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/DateTime}))+ (repeat 10 nil))))))(testing "handle all supported types"(is (= {:global {:distinct-count 5:nil% 0.0}:type {:type/DateTime {:earliest "1970-01-01T00:00:01.234Z":latest "2020-07-06T20:25:33.36Z"}}}- (transduce identity- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))- [(java.time.LocalDateTime/of 2013 01 01 20 04 0 0) ; LocalDateTime- 1234 ; int- 1594067133360 ; long- "2007-12-03T10:15:30.00Z" ; string- (java.time.ZonedDateTime/of 2016 01 01 20 04 0 0 java.time.ZoneOffset/UTC)]))))+ (temporal-core-fields+ (transduce identity+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Temporal}))+ [(java.time.LocalDateTime/of 2013 01 01 20 04 0 0) ; LocalDateTime+ 1234 ; int+ 1594067133360 ; long+ "2007-12-03T10:15:30.00Z" ; string+ (java.time.ZonedDateTime/of 2016 01 01 20 04 0 0 java.time.ZoneOffset/UTC)])))))(testing "we respect effective_type"(is (= {:global {:distinct-count 2:nil% 0.0}:type {:type/DateTime {:earliest "1970-01-01T00:00:01.234Z":latest "2007-12-03T10:15:30Z"}}}- (transduce identity- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text :effective_type :type/Temporal}))- ["2007-12-03T10:15:30.00Z" "1970-01-01T00:00:01.234Z"]))))))))))+ (temporal-core-fields+ (transduce identity+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text :effective_type :type/Temporal}))+ ["2007-12-03T10:15:30.00Z" "1970-01-01T00:00:01.234Z"])))))))))))(deftest ^:parallel disambiguate-test(testing "We should correctly disambiguate multiple competing multimethods (DateTime and FK in this case)"@@ -92,40 +108,46 @@:nil% 0.0}:type {:type/DateTime {:earliest "2013-01-01":latest "2018-01-01"}}}- (transduce identity- (fingerprinters/fingerprinter field)- [#t "2013" #t "2018" #t "2015"]))))))+ (temporal-core-fields+ (transduce identity+ (fingerprinters/fingerprinter field)+ [#t "2013" #t "2018" #t "2015"])))))))(deftest ^:parallel fingerprint-numeric-values-test(is (= {:global {:distinct-count 3:nil% 0.0}- :type {:type/Number {:avg 2.0- :min 1.0- :max 3.0- :q1 1.25- :q3 2.75- :sd 1.0}}}+ :type {:type/Number {:avg 2.0+ :min 1.0+ :max 3.0+ :q1 1.25+ :q3 2.75+ :sd 1.0+ :skewness 0.0+ :excess-kurtosis nil+ :mode-fraction (/ 1.0 3.0)+ :top-3-fraction 1.0+ :zero-fraction 0.0}}}(transduce identity(fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Number}))[1.0 2.0 3.0])))(testing "we respect effective_type"- (is (= {:global {:distinct-count 4, :nil% 0.0},- :type {:type/Number {:min 1.0, :q1 1.15, :q3 2.15, :max 2.3, :sd 0.6027713773341707, :avg 1.65}}}- (transduce identity- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text :effective_type :type/Number}))- ["1" "2" "1.3" "2.3"]))))+ (let [result (transduce identity+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text :effective_type :type/Number}))+ ["1" "2" "1.3" "2.3"])]+ (is (= {:distinct-count 4, :nil% 0.0} (:global result)))+ (is (= {:min 1.0 :q1 1.15 :q3 2.15 :max 2.3 :avg 1.65}+ (select-keys (get-in result [:type :type/Number])+ [:min :q1 :q3 :max :avg])))+ (is (some? (get-in result [:type :type/Number :skewness])))+ (is (some? (get-in result [:type :type/Number :mode-fraction])))))(testing "We should robustly survive weird values such as NaN, Infinity, and nil"- (is (= {:global {:distinct-count 7- :nil% 0.25}- :type {:type/Number {:avg 2.0- :min 1.0- :max 3.0- :q1 1.25- :q3 2.75- :sd 1.0}}}- (transduce identity- (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Number}))- [1.0 2.0 3.0 Double/NaN Double/POSITIVE_INFINITY Double/NEGATIVE_INFINITY nil nil])))))+ (let [result (transduce identity+ (fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Number}))+ [1.0 2.0 3.0 Double/NaN Double/POSITIVE_INFINITY Double/NEGATIVE_INFINITY nil nil])]+ (is (= {:distinct-count 7, :nil% 0.25} (:global result)))+ (is (= {:avg 2.0, :min 1.0, :max 3.0, :q1 1.25, :q3 2.75, :sd 1.0}+ (select-keys (get-in result [:type :type/Number])+ [:avg :min :max :q1 :q3 :sd]))))))(deftest ^:parallel fingerprint-string-values-test(is (= {:global {:distinct-count 5@@ -134,7 +156,12 @@:percent-url 0.0:percent-email 0.0:percent-state 0.0- :average-length 6.4}}}+ :average-length 6.4+ :min-length 4.0+ :max-length 9.0+ :mode-fraction 0.2+ :top-3-fraction 0.6+ :percent-blank 0.0}}}(transduce identity(fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text}))["metabase" "more" "like" "metabae" "[1, 2, 3]"])))@@ -145,7 +172,12 @@:percent-url 0.0:percent-email 0.0:percent-state 0.0- :average-length 10.6}}}+ :average-length 10.6+ :min-length 4.0+ :max-length 30.0+ :mode-fraction 0.2+ :top-3-fraction 0.6+ :percent-blank 0.0}}}(transduce identity(fingerprinters/fingerprinter (mi/instance :model/Field {:base_type :type/Text}))["metabase" "more" "like" "metabae" truncated-json])))))diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex bcc47b60fdf..76972a9ef58 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -1309,23 +1309,34 @@" VALUES ('22:00'::time, '9:00'::time, 'Beauty Sleep');")])(mt/with-temp [:model/Database database {:engine :postgres, :details (assoc details :dbname "time_field_test")}](sync/sync-database! database)- (is (= {"start_time" {:global {:distinct-count 1- :nil% 0.0}- :type {:type/DateTime {:earliest "22:00:00"- :latest "22:00:00"}}}- "end_time" {:global {:distinct-count 1- :nil% 0.0}- :type {:type/DateTime {:earliest "09:00:00"- :latest "09:00:00"}}}- "reason" {:global {:distinct-count 1- :nil% 0.0}- :type {:type/Text {:percent-json 0.0- :percent-url 0.0- :percent-email 0.0- :percent-state 0.0- :average-length 12.0}}}}- (t2/select-fn->fn :name :fingerprint :model/Field- :table_id (t2/select-one-pk :model/Table :db_id (u/the-id database))))))))))+ (let [fingerprints (t2/select-fn->fn :name :fingerprint :model/Field+ :table_id (t2/select-one-pk :model/Table :db_id (u/the-id database)))+ ;; Strip extended interestingness stats — this test covers the core TIME fingerprint+ ;; shape (#5911), not the interestingness metrics.+ extended-keys [:hour-distribution :weekday-distribution :skewness+ :mode-fraction :top-3-fraction+ :mode-fraction-by-weekday :mode-fraction-by-hour+ :min-length :max-length :percent-blank]+ trim-type (fn [fp]+ (update fp :type+ (fn [types]+ (update-vals types #(apply dissoc % extended-keys)))))]+ (is (= {"start_time" {:global {:distinct-count 1+ :nil% 0.0}+ :type {:type/DateTime {:earliest "22:00:00"+ :latest "22:00:00"}}}+ "end_time" {:global {:distinct-count 1+ :nil% 0.0}+ :type {:type/DateTime {:earliest "09:00:00"+ :latest "09:00:00"}}}+ "reason" {:global {:distinct-count 1+ :nil% 0.0}+ :type {:type/Text {:percent-json 0.0+ :percent-url 0.0+ :percent-email 0.0+ :percent-state 0.0+ :average-length 12.0}}}}+ (update-vals fingerprints trim-type)))))))));;; ----------------------------------------------------- Other ------------------------------------------------------diff --git a/test/metabase/metabot/stats/categorical_test.clj b/test/metabase/interestingness/chart/categorical_test.cljsimilarity index 98%rename from test/metabase/metabot/stats/categorical_test.cljrename to test/metabase/interestingness/chart/categorical_test.cljindex 18bc184dd41..6eb9bd271b9 100644--- a/test/metabase/metabot/stats/categorical_test.clj+++ b/test/metabase/interestingness/chart/categorical_test.clj@@ -1,8 +1,8 @@-(ns metabase.metabot.stats.categorical-test+(ns metabase.interestingness.chart.categorical-test(:require[clojure.test :refer :all][mb.hawk.assert-exprs.approximately-equal :as =?]- [metabase.metabot.stats.categorical :as categorical]))+ [metabase.interestingness.chart.categorical :as categorical]))(set! *warn-on-reflection* true)diff --git a/test/metabase/metabot/stats/histogram_test.clj b/test/metabase/interestingness/chart/histogram_test.cljsimilarity index 98%rename from test/metabase/metabot/stats/histogram_test.cljrename to test/metabase/interestingness/chart/histogram_test.cljindex d34703058b8..1057aff340f 100644--- a/test/metabase/metabot/stats/histogram_test.clj+++ b/test/metabase/interestingness/chart/histogram_test.clj@@ -1,8 +1,8 @@-(ns metabase.metabot.stats.histogram-test+(ns metabase.interestingness.chart.histogram-test(:require[clojure.test :refer :all][mb.hawk.assert-exprs.approximately-equal :as =?]- [metabase.metabot.stats.histogram :as histogram]))+ [metabase.interestingness.chart.histogram :as histogram]))(set! *warn-on-reflection* true)diff --git a/test/metabase/metabot/stats/outliers_test.clj b/test/metabase/interestingness/chart/outliers_test.cljsimilarity index 97%rename from test/metabase/metabot/stats/outliers_test.cljrename to test/metabase/interestingness/chart/outliers_test.cljindex 513cea9f1d0..013ff8b3313 100644--- a/test/metabase/metabot/stats/outliers_test.clj+++ b/test/metabase/interestingness/chart/outliers_test.clj@@ -1,8 +1,8 @@-(ns metabase.metabot.stats.outliers-test+(ns metabase.interestingness.chart.outliers-test(:require[clojure.test :refer :all][mb.hawk.assert-exprs.approximately-equal :as =?]- [metabase.metabot.stats.outliers :as outliers]))+ [metabase.interestingness.chart.outliers :as outliers]))(set! *warn-on-reflection* true)diff --git a/test/metabase/metabot/stats/repr_test.clj b/test/metabase/interestingness/chart/repr_test.cljsimilarity index 96%rename from test/metabase/metabot/stats/repr_test.cljrename to test/metabase/interestingness/chart/repr_test.cljindex 29cb114e0f8..9a3dd7f0bbc 100644--- a/test/metabase/metabot/stats/repr_test.clj+++ b/test/metabase/interestingness/chart/repr_test.clj@@ -1,13 +1,13 @@-(ns metabase.metabot.stats.repr-test+(ns metabase.interestingness.chart.repr-test(:require[clojure.string :as str][clojure.test :refer :all]- [metabase.metabot.stats.categorical :as categorical]- [metabase.metabot.stats.core :as stats.core]- [metabase.metabot.stats.histogram :as histogram]- [metabase.metabot.stats.repr :as repr]- [metabase.metabot.stats.scatter :as scatter]- [metabase.metabot.stats.time-series :as time-series]+ [metabase.interestingness.chart.categorical :as categorical]+ [metabase.interestingness.chart.histogram :as histogram]+ [metabase.interestingness.chart.repr :as repr]+ [metabase.interestingness.chart.scatter :as scatter]+ [metabase.interestingness.chart.stats :as stats.core]+ [metabase.interestingness.chart.time-series :as time-series][metabase.util :as u]))(set! *warn-on-reflection* true)diff --git a/test/metabase/metabot/stats/scatter_test.clj b/test/metabase/interestingness/chart/scatter_test.cljsimilarity index 98%rename from test/metabase/metabot/stats/scatter_test.cljrename to test/metabase/interestingness/chart/scatter_test.cljindex 019b2d7be53..9fcf9d641de 100644--- a/test/metabase/metabot/stats/scatter_test.clj+++ b/test/metabase/interestingness/chart/scatter_test.clj@@ -1,8 +1,8 @@-(ns metabase.metabot.stats.scatter-test+(ns metabase.interestingness.chart.scatter-test(:require[clojure.test :refer :all][mb.hawk.assert-exprs.approximately-equal :as =?]- [metabase.metabot.stats.scatter :as scatter]))+ [metabase.interestingness.chart.scatter :as scatter]))(set! *warn-on-reflection* true)diff --git a/test/metabase/metabot/stats/core_test.clj b/test/metabase/interestingness/chart/stats_test.cljsimilarity index 97%rename from test/metabase/metabot/stats/core_test.cljrename to test/metabase/interestingness/chart/stats_test.cljindex 8fc7f4e8f77..ff822797518 100644--- a/test/metabase/metabot/stats/core_test.clj+++ b/test/metabase/interestingness/chart/stats_test.clj@@ -1,7 +1,7 @@-(ns metabase.metabot.stats.core-test+(ns metabase.interestingness.chart.stats-test(:require[clojure.test :refer :all]- [metabase.metabot.stats.core :as stats.core]))+ [metabase.interestingness.chart.stats :as stats.core]))(set! *warn-on-reflection* true)diff --git a/test/metabase/metabot/stats/time_series_test.clj b/test/metabase/interestingness/chart/time_series_test.cljsimilarity index 99%rename from test/metabase/metabot/stats/time_series_test.cljrename to test/metabase/interestingness/chart/time_series_test.cljindex 7cec5b0af2e..9aeab74a6c3 100644--- a/test/metabase/metabot/stats/time_series_test.clj+++ b/test/metabase/interestingness/chart/time_series_test.clj@@ -1,10 +1,10 @@-(ns metabase.metabot.stats.time-series-test+(ns metabase.interestingness.chart.time-series-test(:require[clojure.string :as str][clojure.test :refer :all][mb.hawk.assert-exprs.approximately-equal :as =?]- [metabase.metabot.stats.time-series :as time-series]- [metabase.metabot.stats.util :as stats.u]))+ [metabase.interestingness.chart.time-series :as time-series]+ [metabase.interestingness.chart.util :as stats.u]))(set! *warn-on-reflection* true)diff --git a/test/metabase/metabot/stats/util_test.clj b/test/metabase/interestingness/chart/util_test.cljsimilarity index 99%rename from test/metabase/metabot/stats/util_test.cljrename to test/metabase/interestingness/chart/util_test.cljindex a0c290ed462..83d60646e78 100644--- a/test/metabase/metabot/stats/util_test.clj+++ b/test/metabase/interestingness/chart/util_test.clj@@ -1,8 +1,8 @@-(ns metabase.metabot.stats.util-test+(ns metabase.interestingness.chart.util-test(:require[clojure.test :refer :all][mb.hawk.assert-exprs.approximately-equal :as =?]- [metabase.metabot.stats.util :as stats.u]))+ [metabase.interestingness.chart.util :as stats.u]))(set! *warn-on-reflection* true)diff --git a/test/metabase/interestingness/chart_test.clj b/test/metabase/interestingness/chart_test.cljnew file mode 100644index 00000000000..5f613249a36--- /dev/null+++ b/test/metabase/interestingness/chart_test.clj@@ -0,0 +1,75 @@+(ns metabase.interestingness.chart-test+ (:require+ [clojure.test :refer :all]+ [metabase.interestingness.chart :as chart]+ [metabase.interestingness.core :as interestingness]))++(set! *warn-on-reflection* true)++(defn- time-series-config+ "Build a time-series chart-config with the given y_values."+ [ys]+ {:display_type "line"+ :title "Test"+ :series {"s1" {:x {:name "Date" :type "datetime"}+ :y {:name "Value" :type "number"}+ :display_name "Value"+ :x_values (mapv #(format "2024-01-%02d" (inc %)) (range (count ys)))+ :y_values (mapv double ys)}}})++(defn- categorical-config+ [categories values]+ {:display_type "bar"+ :title "Test"+ :series {"s1" {:x {:name "Category" :type "string"}+ :y {:name "Count" :type "number"}+ :display_name "Count"+ :x_values (vec categories)+ :y_values (mapv double values)}}})++(deftest ^:parallel re-exported-from-core-test+ (testing "chart-interestingness is re-exported from interestingness.core"+ (is (some? (resolve 'metabase.interestingness.core/chart-interestingness)))+ (is (number? (interestingness/chart-interestingness (time-series-config (range 20)))))))++(deftest ^:parallel score-in-unit-interval-test+ (testing "chart-interestingness always returns a value in [0, 1]"+ (doseq [cfg [(time-series-config (range 30))+ (time-series-config (repeat 30 5))+ (categorical-config ["a" "b" "c"] [10 20 30])+ (categorical-config ["only"] [42])]]+ (let [score (chart/chart-interestingness cfg)]+ (is (<= 0.0 score 1.0) (str "score out of range: " score))))))++(deftest ^:parallel flat-measure-is-uninteresting-test+ (testing "a chart with zero-variance y_values scores 0"+ (let [cfg (time-series-config (repeat 30 7))]+ (is (zero? (chart/chart-interestingness cfg))))))++(deftest ^:parallel single-valued-dimension-is-uninteresting-test+ (testing "a chart with a single x value scores 0"+ (let [cfg (categorical-config ["only"] [42])]+ (is (zero? (chart/chart-interestingness cfg))))))++(deftest ^:parallel trend-beats-flat-test+ (testing "a strongly-increasing time series scores higher than a nearly-flat one"+ (let [trend (time-series-config (range 30))+ noisy (time-series-config (map #(+ 100 (rem % 3)) (range 30)))]+ (is (> (chart/chart-interestingness trend)+ (chart/chart-interestingness noisy))))))++(deftest ^:parallel outlier-elevates-score-test+ (testing "injecting a large outlier raises the score vs. a near-constant series"+ (let [boring (vec (repeat 29 10))+ with-out (conj (vec (repeat 29 10)) 10000)]+ (is (> (chart/chart-interestingness (time-series-config with-out))+ (chart/chart-interestingness (time-series-config boring)))))))++(deftest ^:parallel caller-supplied-stats-test+ (testing "2-arity form accepts pre-computed stats (no recomputation)"+ (let [cfg (time-series-config (range 30))+ stats {:chart-type :time-series+ :series-count 0+ :series {}}+ score (chart/chart-interestingness cfg stats)]+ (is (<= 0.0 score 1.0)))))diff --git a/test/metabase/interestingness/core_test.clj b/test/metabase/interestingness/core_test.cljnew file mode 100644index 00000000000..7b974df713a--- /dev/null+++ b/test/metabase/interestingness/core_test.clj@@ -0,0 +1,25 @@+(ns metabase.interestingness.core-test+ (:require+ [clojure.test :refer [deftest is]]+ [metabase.interestingness.core :as interestingness]))++;;; Tests here cover only the public facade — the `compute interestingness for X`+;;; user-facing legos. The underlying composition machinery is tested in+;;; `metabase.interestingness.impl-test` and the scorers in+;;; `metabase.interestingness.dimension-test`.++(deftest ^:parallel dimension-interestingness-scores-normalized-field-test+ (is (>= (interestingness/dimension-interestingness+ {:semantic-type :type/CreationTimestamp+ :base-type :type/DateTime+ :fingerprint {:global {:distinct-count 5000 :nil% 0.0}+ :type {:type/DateTime {:earliest "2022-01-01"+ :latest "2024-12-31"}}}})+ 0.7)))++(deftest ^:parallel dimension-interestingness-accepts-snake-case-test+ (is (<= (interestingness/dimension-interestingness+ {:semantic_type :type/PK+ :base_type :type/Integer+ :fingerprint {:global {:distinct-count 1000 :nil% 0.0}}})+ 0.1)))diff --git a/test/metabase/interestingness/dimension_test.clj b/test/metabase/interestingness/dimension_test.cljnew file mode 100644index 00000000000..3dafc86cbae--- /dev/null+++ b/test/metabase/interestingness/dimension_test.clj@@ -0,0 +1,129 @@+(ns metabase.interestingness.dimension-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.interestingness.dimension :as dim]))++;;; -------------------------------------------------- cardinality --------------------------------------------------++(deftest ^:parallel cardinality-test+ (testing "constant field (1 distinct) scores 0.0"+ (is (= 0.0 (:score (dim/cardinality {:fingerprint {:global {:distinct-count 1}}})))))+ (testing "low cardinality (10 = top of ramp to sweet spot) scores high"+ (let [score (:score (dim/cardinality {:fingerprint {:global {:distinct-count 10}}}))]+ (is (>= score 0.8))))+ (testing "moderate cardinality (50 = past sweet spot, declining) scores moderately"+ (let [score (:score (dim/cardinality {:fingerprint {:global {:distinct-count 50}}}))]+ (is (>= score 0.7))))+ (testing "high cardinality scores lower"+ (let [score (:score (dim/cardinality {:fingerprint {:global {:distinct-count 5000}}}))]+ (is (< score 0.5))))+ (testing "missing fingerprint returns 0.5"+ (is (= 0.5 (:score (dim/cardinality {}))))+ (is (= 0.5 (:score (dim/cardinality {:fingerprint {}}))))))++;;; -------------------------------------------------- type-bonus --------------------------------------------------++(deftest ^:parallel type-bonus-test+ (testing "listed bonus types score 1.0"+ (doseq [sem-type [:type/CreationTimestamp+ :type/Country ; via :type/Address+ :type/ZipCode ; via :type/Address+ :type/Income ; via :type/Currency+ :type/Price ; via :type/Currency+ :type/Birthdate+ :type/Title+ :type/Quantity+ :type/Share+ :type/Score+ :type/JoinDate ; via :type/JoinTemporal+ :type/CancelationTimestamp ; via :type/CancelationTemporal+ :type/Company+ :type/Subscription+ :type/Owner]]+ (is (= 1.0 (:score (dim/type-bonus {:semantic-type sem-type})))+ (str sem-type " should score 1.0"))))+ (testing "non-listed types score 0.5"+ (doseq [sem-type [:type/Category :type/Name :type/Number :type/Text nil]]+ (is (= 0.5 (:score (dim/type-bonus {:semantic-type sem-type})))+ (str sem-type " should score 0.5"))))+ (testing "base-type alone (no semantic-type) does not earn a bonus"+ (is (= 0.5 (:score (dim/type-bonus {:base-type :type/DateTime}))))+ (is (= 0.5 (:score (dim/type-bonus {:base-type :type/Boolean})))))+ (testing "no semantic type returns 0.5"+ (is (= 0.5 (:score (dim/type-bonus {}))))))++;;; -------------------------------------------------- temporal-range --------------------------------------------------++(deftest ^:parallel temporal-range-test+ (testing "multi-year range scores high"+ (let [score (:score (dim/temporal-range+ {:fingerprint {:type {:type/DateTime {:earliest "2020-01-01" :latest "2024-01-01"}}}}))]+ (is (>= score 0.9))))+ (testing "same-day range scores low"+ (let [score (:score (dim/temporal-range+ {:fingerprint {:type {:type/DateTime {:earliest "2024-01-01" :latest "2024-01-01"}}}}))]+ (is (<= score 0.2))))+ (testing "short range (few days) scores moderate"+ (let [score (:score (dim/temporal-range+ {:fingerprint {:type {:type/DateTime {:earliest "2024-01-01" :latest "2024-01-10"}}}}))]+ (is (> score 0.3))+ (is (< score 0.7))))+ (testing "non-temporal field returns 0.5"+ (is (= 0.5 (:score (dim/temporal-range {})))))+ (testing "missing bounds returns 0.5"+ (is (= 0.5 (:score (dim/temporal-range+ {:fingerprint {:type {:type/DateTime {:earliest nil :latest nil}}}}))))))++;;; -------------------------------------------------- text-structure --------------------------------------------------++(deftest ^:parallel text-structure-test+ (testing "high JSON percentage scores low"+ (let [score (:score (dim/text-structure+ {:fingerprint {:type {:type/Text {:percent-json 0.95}}}}))]+ (is (<= score 0.15))))+ (testing "high URL percentage scores low"+ (let [score (:score (dim/text-structure+ {:fingerprint {:type {:type/Text {:percent-url 0.95}}}}))]+ (is (<= score 0.2))))+ (testing "long average length scores low"+ (let [score (:score (dim/text-structure+ {:fingerprint {:type {:type/Text {:average-length 150}}}}))]+ (is (<= score 0.25))))+ (testing "moderate length scores low (soft penalty)"+ (let [score (:score (dim/text-structure+ {:fingerprint {:type {:type/Text {:average-length 60}}}}))]+ (is (> score 0.1))+ (is (< score 0.3))))+ (testing "very long text (> 100 chars) scores near-zero but isn't a hard-zero"+ (let [score (:score (dim/text-structure+ {:fingerprint {:type {:type/Text {:average-length 200}}}}))]+ (is (< score 0.1))+ (is (> score 0.0))))+ (testing "short text scores high"+ (let [score (:score (dim/text-structure+ {:fingerprint {:type {:type/Text {:average-length 10}}}}))]+ (is (>= score 0.7))))+ (testing "high email percentage scores low (PII safety net for missed classifier)"+ (let [result (dim/text-structure+ {:fingerprint {:type {:type/Text {:percent-email 0.95}}}})]+ (is (<= (:score result) 0.2))+ (is (re-find #"email" (:reason result)))))+ (testing "high state percentage is flagged (suggests map viz, not breakout)"+ (let [result (dim/text-structure+ {:fingerprint {:type {:type/Text {:percent-state 0.95}}}})]+ (is (<= (:score result) 0.4))+ (is (re-find #"state" (:reason result)))))+ (testing "non-text field returns 0.5"+ (is (= 0.5 (:score (dim/text-structure {}))))))++;;; -------------------------------------------------- text-structure percent-blank --------------------------------------------------++(deftest ^:parallel text-structure-blank-test+ (testing "mostly-blank text scores very low"+ (let [score (:score (dim/text-structure+ {:fingerprint {:type {:type/Text {:percent-blank 0.9}}}}))]+ (is (<= score 0.15))))+ (testing "low percent-blank doesn't trigger the penalty"+ (let [score (:score (dim/text-structure+ {:fingerprint {:type {:type/Text {:percent-blank 0.1 :average-length 10}}}}))]+ (is (>= score 0.7)))))diff --git a/test/metabase/interestingness/impl_test.clj b/test/metabase/interestingness/impl_test.cljnew file mode 100644index 00000000000..7adf5415020--- /dev/null+++ b/test/metabase/interestingness/impl_test.clj@@ -0,0 +1,202 @@+(ns metabase.interestingness.impl-test+ (:require+ [clojure.test :refer [are deftest is testing]]+ [metabase.interestingness.impl :as impl]))++;;; -------------------------------------------------- type-penalty --------------------------------------------------++(deftest ^:parallel type-penalty-test+ (testing "penalized types score 0.0"+ (are [sem-type reason-substr]+ (let [{:keys [score reason]} (impl/type-penalty {:semantic-type sem-type})]+ (and (= 0.0 score) (re-find (re-pattern reason-substr) reason)))+ :type/PK "primary key"+ :type/SerializedJSON "structured blob"+ :type/JSON "structured blob"+ :type/XML "structured blob"+ :type/Array "structured blob"+ :type/Dictionary "structured blob"+ :type/UpdatedTimestamp "updated timestamp"+ :type/DeletionTimestamp "deletion timestamp"))++ (testing "non-penalized types score 1.0 (including FK — x-ray templates use FK columns)"+ (are [sem-type]+ (= 1.0 (:score (impl/type-penalty {:semantic-type sem-type})))+ :type/Category+ :type/Name+ :type/CreationTimestamp+ :type/Number+ :type/FK+ nil))++ (testing "nil semantic type scores 1.0"+ (is (= 1.0 (:score (impl/type-penalty {}))))))++;;; -------------------------------------------------- score-field composition --------------------------------------------------++(defn- constant-scorer [score reason]+ (fn [_field] {:score score :reason reason}))++(deftest ^:parallel score-field-test+ (testing "single scorer"+ (let [result (impl/score-field+ {(constant-scorer 0.8 "good") 1.0}+ {:semantic-type :type/Category})]+ (is (= 0.8 (:score result)))+ (is (map? (:scores result)))+ (is (= {:semantic-type :type/Category} (:field result)))))++ (testing "weighted average of two scorers"+ (let [result (impl/score-field+ {(constant-scorer 1.0 "high") 0.75+ (constant-scorer 0.5 "mid") 0.25}+ {})]+ (is (= 0.875 (:score result)))))++ (testing "weights don't need to sum to 1"+ (let [result (impl/score-field+ {(constant-scorer 1.0 "a") 3.0+ (constant-scorer 0.5 "b") 1.0}+ {})]+ (is (= 0.875 (:score result)))))++ (testing "hard-zero from any scorer clamps total to at most 0.1"+ (let [result (impl/score-field+ {(constant-scorer 1.0 "high") 0.75+ (constant-scorer 0.0 "gate") 0.25}+ {})]+ (is (<= (:score result) 0.1))))++ (testing "empty scorer map returns 0.5"+ (is (= 0.5 (:score (impl/score-field {} {}))))))++(deftest ^:parallel compose-test+ (testing "compose returns a callable scorer"+ (let [composed (impl/compose+ {(constant-scorer 0.8 "a") 1.0+ (constant-scorer 0.6 "b") 1.0})+ result (composed {})]+ (is (= 0.7 (:score result))))))++(deftest ^:parallel apply-cutoff-test+ (testing "filters below threshold"+ (let [scored [{:score 0.8 :field :a}+ {:score 0.3 :field :b}+ {:score 0.5 :field :c}]]+ (is (= [{:score 0.8 :field :a} {:score 0.5 :field :c}]+ (impl/apply-cutoff 0.5 scored)))))++ (testing "empty input returns empty"+ (is (empty? (impl/apply-cutoff 0.5 [])))))++(deftest ^:parallel score-and-filter-test+ (testing "scores, filters, and sorts descending"+ (let [fields [{:name "good"} {:name "bad"} {:name "ok"}]+ name-scorer (fn [field]+ (let [len (count (:name field))]+ {:score (/ len 10.0) :reason (str len " chars")}))+ results (impl/score-and-filter+ {name-scorer 1.0} fields 0.25)]+ (is (= 2 (count results)))+ (is (= "good" (-> results first :field :name)))+ (is (>= (-> results first :score) (-> results second :score))))))++;;; -------------------------------------------------- nullness --------------------------------------------------++(deftest ^:parallel nullness-test+ (testing "0% null scores 1.0"+ (is (= 1.0 (:score (impl/nullness {:fingerprint {:global {:nil% 0.0}}})))))++ (testing "100% null scores 0.0"+ (is (= 0.0 (:score (impl/nullness {:fingerprint {:global {:nil% 1.0}}})))))++ (testing "50% null scores 0.5"+ (is (= 0.5 (:score (impl/nullness {:fingerprint {:global {:nil% 0.5}}})))))++ (testing "missing fingerprint returns 0.5"+ (is (= 0.5 (:score (impl/nullness {}))))))++;;; -------------------------------------------------- numeric-variance --------------------------------------------------++(deftest ^:parallel numeric-variance-test+ (testing "zero sd scores 0.0"+ (is (= 0.0 (:score (impl/numeric-variance+ {:fingerprint {:type {:type/Number {:sd 0.0 :avg 10.0}}}})))))++ (testing "q1 = q3 scores low"+ (is (<= (:score (impl/numeric-variance+ {:fingerprint {:type {:type/Number {:q1 5.0 :q3 5.0}}}}))+ 0.1)))++ (testing "healthy spread scores high"+ (let [score (:score (impl/numeric-variance+ {:fingerprint {:type {:type/Number {:sd 10.0 :avg 50.0 :min 0 :max 100}}}}))]+ (is (> score 0.3))))++ (testing "non-numeric field returns 0.5"+ (is (= 0.5 (:score (impl/numeric-variance {}))))+ (is (= 0.5 (:score (impl/numeric-variance {:fingerprint {:type {:type/Text {}}}}))))))++;;; -------------------------------------------------- distribution-shape --------------------------------------------------++(deftest ^:parallel distribution-shape-test+ (testing "no distribution data returns 0.5"+ (is (= 0.5 (:score (impl/distribution-shape {}))))+ (is (= 0.5 (:score (impl/distribution-shape+ {:fingerprint {:type {:type/Number {:avg 10 :sd 2}}}})))))++ (testing "symmetric low-dominance distribution scores high"+ (let [score (:score (impl/distribution-shape+ {:fingerprint {:type {:type/Number {:skewness 0.1 :mode-fraction 0.1}}}}))]+ (is (>= score 0.9))))++ (testing "heavy skew penalizes score"+ (let [score (:score (impl/distribution-shape+ {:fingerprint {:type {:type/Number {:skewness 3.0 :mode-fraction 0.1}}}}))]+ (is (<= score 0.5))))++ (testing "high mode-dominance scores very low"+ (let [score (:score (impl/distribution-shape+ {:fingerprint {:type {:type/Text {:mode-fraction 0.97}}}}))]+ (is (<= score 0.1))))++ (testing "80% dominance scores low but not critical"+ (let [score (:score (impl/distribution-shape+ {:fingerprint {:type {:type/Text {:mode-fraction 0.85}}}}))]+ (is (>= score 0.15))+ (is (<= score 0.3))))++ (testing "combined signals: worst-of-two wins"+ (let [score (:score (impl/distribution-shape+ {:fingerprint {:type {:type/Number {:skewness 0.1 :mode-fraction 0.95}}}}))]+ (is (<= score 0.1))))++ (testing "works on text fields via type/Text fingerprint"+ (let [score (:score (impl/distribution-shape+ {:fingerprint {:type {:type/Text {:mode-fraction 0.3}}}}))]+ (is (>= score 0.9))))++ (testing "extreme excess kurtosis penalizes score"+ (let [score (:score (impl/distribution-shape+ {:fingerprint {:type {:type/Number {:excess-kurtosis 20.0}}}}))]+ (is (<= score 0.2))))++ (testing "near-normal kurtosis scores high"+ (let [score (:score (impl/distribution-shape+ {:fingerprint {:type {:type/Number {:excess-kurtosis 0.2}}}}))]+ (is (>= score 0.9))))++ (testing "high top-3-fraction penalizes score"+ (let [score (:score (impl/distribution-shape+ {:fingerprint {:type {:type/Text {:mode-fraction 0.4 :top-3-fraction 0.995}}}}))]+ (is (<= score 0.15))))++ (testing "high zero-fraction (numeric) penalizes score"+ (let [score (:score (impl/distribution-shape+ {:fingerprint {:type {:type/Number {:zero-fraction 0.96}}}}))]+ (is (<= score 0.1))))++ (testing "temporal mode-fraction penalizes dumping-ground timestamps"+ (let [score (:score (impl/distribution-shape+ {:fingerprint {:type {:type/DateTime {:mode-fraction 0.95}}}}))]+ (is (<= score 0.1)))))diff --git a/test/metabase/metabot/agent/tool_output_test.clj b/test/metabase/metabot/agent/tool_output_test.cljindex 5721e41d342..eda5615e790 100644--- a/test/metabase/metabot/agent/tool_output_test.clj+++ b/test/metabase/metabot/agent/tool_output_test.clj@@ -11,6 +11,7 @@[metabase.metabot.tools.metadata :as metadata-tools][metabase.metabot.tools.resources :as resource-tools][metabase.metabot.tools.search :as search-tools]+ [metabase.search.core :as search][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.warehouse-schema.models.field-values :as field-values]@@ -196,6 +197,7 @@(deftest search-tool-structured-output-formats-correctly-test(testing "search tool :structured-output formats to clean XML via format-structured-result"(mt/test-driver :h2+ (search/init-index! {:force-reset? false :re-populate? true})(mt/with-current-user (mt/user->id :crowberto)(let [result (search-tools/search-tool{:semantic_queries ["orders"]@@ -206,6 +208,7 @@(deftest search-tool-with-models-structured-output-test(testing "search tool with model results formats correctly"(mt/test-driver :h2+ (search/init-index! {:force-reset? false :re-populate? true})(mt/with-current-user (mt/user->id :crowberto)(let [model-query (-> (lib/query (mt/metadata-provider)(lib.metadata/table (mt/metadata-provider) (mt/id :orders)))@@ -215,6 +218,7 @@:database_id (mt/id):name "Searchable Test Model":type :model}]+ (search/init-index! {:force-reset? false :re-populate? true})(let [result (search-tools/search-tool{:semantic_queries ["Searchable Test Model"]:keyword_queries ["Searchable Test Model"]diff --git a/test/metabase/metabot/tools/field_stats_test.clj b/test/metabase/metabot/tools/field_stats_test.cljindex 29fa7264cc0..ad8502eb618 100644--- a/test/metabase/metabot/tools/field_stats_test.clj+++ b/test/metabase/metabot/tools/field_stats_test.clj@@ -28,24 +28,46 @@{:entity-type "table", :entity-id people-id, :field-id state-id, :limit 5}))))(testing "Getting statistics and values for table fields works."(mt/as-admin+ ;; Skewness is a derived double whose last few digits vary across JVMs/platforms,+ ;; so strip it from the exact-equality check and assert approximate value separately.+ (let [birth-date-result (metabot.tools.field-stats/field-values+ {:entity-type "table", :entity-id people-id, :field-id birth-date-id, :limit 5})+ birth-date-skewness (get-in birth-date-result+ [:structured-output :value_metadata :statistics :skewness])]+ (is (some? birth-date-skewness))+ (is (< (abs (- birth-date-skewness -0.00557870227770)) 1e-6)+ "skewness should be approximately -0.00557870227770"))(are [table-id field-id value-metadata](= {:structured-output {:result-type :field-metadata:field_id field-id:value_metadata value-metadata}}- (metabot.tools.field-stats/field-values- {:entity-type "table", :entity-id table-id, :field-id field-id, :limit 5}))+ (let [result (metabot.tools.field-stats/field-values+ {:entity-type "table", :entity-id table-id, :field-id field-id, :limit 5})]+ ;; strip skewness to avoid platform-dependent floating-point mismatch+ (cond-> result+ (get-in result [:structured-output :value_metadata :statistics :skewness])+ (update-in [:structured-output :value_metadata :statistics] dissoc :skewness))))people-id birth-date-id {:statistics- {:distinct-count 2308- :percent-null 0.0- :earliest "1958-04-26"- :latest "2000-04-03"}}+ {:distinct-count 2308+ :percent-null 0.0+ :earliest "1958-04-26"+ :latest "2000-04-03"+ :hour-distribution nil+ :mode-fraction 0.0012+ :top-3-fraction 0.0032+ :weekday-distribution [0.15 0.1304 0.1416 0.1372 0.156 0.1516 0.1332]}}people-id state-id {:statistics {:distinct-count 49:percent-null 0.0:percent-json 0.0:percent-url 0.0:percent-email 0.0:percent-state 1.0- :average-length 2.0}+ :average-length 2.0+ :max-length 2.0+ :min-length 2.0+ :mode-fraction 0.0776+ :top-3-fraction 0.1624+ :percent-blank 0.0}:field_values ["AK" "AL" "AR" "AZ" "CA"]}products-id category-id {:statistics {:distinct-count 4:percent-null 0.0@@ -53,7 +75,12 @@:percent-url 0.0:percent-email 0.0:percent-state 0.0- :average-length 6.375}+ :average-length 6.375+ :max-length 9.0+ :min-length 5.0+ :mode-fraction 0.27+ :top-3-fraction 0.79+ :percent-blank 0.0}:field_values ["Doohickey" "Gadget" "Gizmo" "Widget"]})))))(deftest field-values-model-testdiff --git a/test/metabase/sample_data/impl_test.clj b/test/metabase/sample_data/impl_test.cljindex 97c7a75088b..51637a213a3 100644--- a/test/metabase/sample_data/impl_test.clj+++ b/test/metabase/sample_data/impl_test.clj@@ -97,7 +97,12 @@:percent-url 0.0:percent-email 0.0:percent-state 0.0- :average-length 13.532}}}+ :average-length 13.532+ :max-length 23.0+ :min-length 7.0+ :mode-fraction 4.0E-4+ :top-3-fraction 0.0012+ :percent-blank 0.0}}}:base_type :type/Text}(-> (field db "PEOPLE" "NAME");; it should be `nil` after sync but get set to `search` by the auto-inference. We only set `list` indiff --git a/test/metabase/sync/analyze/interestingness_test.clj b/test/metabase/sync/analyze/interestingness_test.cljnew file mode 100644index 00000000000..1c56f672864--- /dev/null+++ b/test/metabase/sync/analyze/interestingness_test.clj@@ -0,0 +1,30 @@+(ns metabase.sync.analyze.interestingness-test+ (:require+ [clojure.test :refer [deftest is]]+ [metabase.interestingness.core :as interestingness]+ [metabase.interestingness.dimension :as dim]))++;;; Smoke tests for the canonical weight profiles. The sync step itself is verified+;;; end-to-end via `automagic_dashboards` integration tests (which fingerprint + score+;;; real tables). Here we just pin down the profile shape and directional behavior.++(deftest ^:parallel canonical-dimension-weights-shape-test+ (is (map? dim/canonical-dimension-weights))+ (is (every? fn? (keys dim/canonical-dimension-weights)))+ (is (every? pos? (vals dim/canonical-dimension-weights))))++(deftest ^:parallel dimension-interestingness-kills-pks-test+ (let [result (interestingness/dimension-interestingness+ {:semantic_type :type/PK+ :base_type :type/Integer+ :fingerprint {:global {:distinct-count 1000 :nil% 0.0}}})]+ (is (<= result 0.1))))++(deftest ^:parallel dimension-interestingness-rewards-temporal-test+ (let [result (interestingness/dimension-interestingness+ {:semantic_type :type/CreationTimestamp+ :base_type :type/DateTime+ :fingerprint {:global {:distinct-count 5000 :nil% 0.0}+ :type {:type/DateTime {:earliest "2022-01-01"+ :latest "2024-12-31"}}}})]+ (is (>= result 0.7))))diff --git a/test/metabase/sync/analyze_test.clj b/test/metabase/sync/analyze_test.cljindex d94b44c4700..892f69414ca 100644--- a/test/metabase/sync/analyze_test.clj+++ b/test/metabase/sync/analyze_test.clj@@ -225,7 +225,7 @@:model/Field _ (fake-field table)](let [results (analyze-table! table)](testing "has the steps performed"- (is (= ["fingerprint-fields" "classify-fields" "classify-tables"]+ (is (= ["fingerprint-fields" "classify-fields" "classify-tables" "score-interestingness"](->> results :steps (map first)))))(testing "has start and finish times"(is (seq (select-keys results [:start-time :end-time]))))))))diff --git a/test/metabase/sync/sync_test.clj b/test/metabase/sync/sync_test.cljindex 035c8f9655c..3f21902771e 100644--- a/test/metabase/sync/sync_test.clj+++ b/test/metabase/sync/sync_test.clj@@ -113,7 +113,8 @@(assoc :fields (for [field (t2/select :model/Field, :table_id (:id table), {:order-by [:name]})](into {} (-> field(update :fingerprint map?)- (update :fingerprint_version (complement zero?))))))+ (update :fingerprint_version (complement zero?))+ (update :dimension_interestingness double?)))))tu/boolean-ids-and-timestamps)))(defn- table-defaults []@@ -132,24 +133,26 @@(defn- field-defaults [](merge(mt/object-defaults :model/Field)- {:created_at true- :fingerprint false- :fingerprint_version false- :fk_target_field_id false+ {:created_at true+ :fingerprint false+ :fingerprint_version false+ :fk_target_field_id false:database_is_auto_increment false- :id true- :last_analyzed false- :parent_id false- :position 0- :json_unfolding false- :table_id true- :updated_at true}))+ :id true+ :last_analyzed false+ :parent_id false+ :position 0+ :json_unfolding false+ :table_id true+ :updated_at true+ :dimension_interestingness false}))(defn- field-defaults-with-fingerprint [](assoc (field-defaults)- :last_analyzed true- :fingerprint_version true- :fingerprint true))+ :last_analyzed true+ :fingerprint_version true+ :fingerprint true+ :dimension_interestingness true))(defn- field:movie-id [](mergediff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.cljindex 0eee595e35b..9f7d6e2b0e2 100644--- a/test/metabase/warehouse_schema_rest/api/table_test.clj+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj@@ -90,7 +90,7 @@field[:created_at :fingerprint :fingerprint_version :fk_target_field_id :id :last_analyzed :updated_at:database_required :database_is_auto_increment :database_is_pk :database_is_generated :database_is_nullable- :entity_id])))+ :entity_id :dimension_interestingness])))(deftest ^:parallel list-table-test(testing "GET /api/table"diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex c80ac3e959b..1c1d802424f 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -116,7 +116,7 @@(select-keysfield[:updated_at :id :created_at :last_analyzed :fingerprint :fingerprint_version :fk_target_field_id- :position]))))+ :position :dimension_interestingness]))))(defn- card-with-native-query [card-name & {:as kvs}](merge