Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Replace storing EDN with JSON in explorations (#81745)

Nathan Voxlandd6cefe359446
diff --git a/resources/migrations/065/20260930_explorations_clear_edn_columns.yaml b/resources/migrations/065/20260930_explorations_clear_edn_columns.yaml
new file mode 100644
index 00000000000..3bb508f0661
--- /dev/null
+++ b/resources/migrations/065/20260930_explorations_clear_edn_columns.yaml
@@ -0,0 +1,106 @@
+## Clear the explorations columns that held Clojure data as EDN, now that they hold JSON
+##
+## The rollback is deliberately empty: explorations is unreleased, so no build that reads these
+## columns as EDN is in anyone's hands, and the values are gone either way.
+
+databaseChangeLog:
+ - changeSet:
+ id: v65.expljsona
+ author: nvoxland
+ comment: Clear exploration_thread.query_plan_transcript, now that it holds JSON
+ changes:
+ - update:
+ tableName: exploration_thread
+ columns:
+ - column:
+ name: query_plan_transcript
+ value: "NULL"
+ rollback: # the EDN this held is gone; there is nothing to restore
+
+ - changeSet:
+ id: v65.expljsonb
+ author: nvoxland
+ comment: Clear exploration_thread.data_access_token, now that it holds JSON
+ changes:
+ - update:
+ tableName: exploration_thread
+ columns:
+ - column:
+ name: data_access_token
+ value: "NULL"
+ rollback: # the EDN this held is gone; there is nothing to restore
+
+ - changeSet:
+ id: v65.expljsonc
+ author: nvoxland
+ comment: Clear exploration_query.data_access_token, now that it holds JSON
+ changes:
+ - update:
+ tableName: exploration_query
+ columns:
+ - column:
+ name: data_access_token
+ value: "NULL"
+ rollback: # the EDN this held is gone; there is nothing to restore
+
+ - changeSet:
+ id: v65.expljsond
+ author: nvoxland
+ comment: Clear stored_result.data_access_token, now that it holds JSON
+ changes:
+ - update:
+ tableName: stored_result
+ columns:
+ - column:
+ name: data_access_token
+ value: "NULL"
+ rollback: # the EDN this held is gone; there is nothing to restore
+
+ - changeSet:
+ id: v65.expljsone
+ author: nvoxland
+ comment: Clear exploration_query_result.chart_stats, now that it holds JSON
+ changes:
+ - update:
+ tableName: exploration_query_result
+ columns:
+ - column:
+ name: chart_stats
+ value: "NULL"
+ rollback: # the EDN this held is gone; there is nothing to restore
+
+ - changeSet:
+ id: v65.expljsonf
+ author: nvoxland
+ comment: Correct the exploration_query_result.chart_stats remarks, which still said EDN
+ changes:
+ - setColumnRemarks:
+ tableName: exploration_query_result
+ columnName: chart_stats
+ columnDataType: ${text.type}
+ remarks: JSON-encoded raw stats on the chart
+ rollback: # a column comment; there is nothing to restore
+
+ - changeSet:
+ id: v65.expljsong
+ author: nvoxland
+ comment: Correct the stored_result.data_access_token remarks, which still said EDN
+ changes:
+ - setColumnRemarks:
+ tableName: stored_result
+ columnName: data_access_token
+ columnDataType: ${text.type}
+ remarks: JSON-encoded effective-data-access token (sandbox/impersonation/routing fingerprint) for the creator's lens this snapshot was computed under. Compared against a viewer's token to gate cached reads. NULL means creator+admin only.
+ rollback: # a column comment; there is nothing to restore
+
+ - changeSet:
+ id: v65.expljsonh
+ author: nvoxland
+ comment: Correct the exploration_thread.query_plan_transcript remarks, which still said EDN
+ changes:
+ - setColumnRemarks:
+ tableName: exploration_thread
+ columnName: query_plan_transcript
+ columnDataType: ${text.type}
+ remarks: JSON-encoded transcript of the planning run (planner, outcome, validation errors, final plan) used to materialize this thread's exploration_query rows. Debug-only — not surfaced to the user.
+ rollback: # a column comment; there is nothing to restore
diff --git a/src/metabase/app_db/encryption.clj b/src/metabase/app_db/encryption.clj
index 15d5e7e620c..a5ea6e567c2 100644
--- a/src/metabase/app_db/encryption.clj
+++ b/src/metabase/app_db/encryption.clj
@@ -29,7 +29,7 @@
(cond-> v
(instance? Blob v) blob->bytes))
-;; All columns whose whole value is encrypted at rest (via `mi/transform-encrypted-json`, or the encrypted-text/EDN
+;; All columns whose whole value is encrypted at rest (via `mi/transform-encrypted-json`, or the encrypted-text/JSON
;; transforms in explorations). The on-disk format is `encrypt(string)`, so rotating the key only requires decrypting
;; the raw value with the current key and re-encrypting the resulting string. We list raw table names (not models) so
;; this also works for enterprise models that aren't loaded in every edition.
diff --git a/src/metabase/explorations/models/exploration_query_result.clj b/src/metabase/explorations/models/exploration_query_result.clj
index b87c35e9e10..42c3787d9f8 100644
--- a/src/metabase/explorations/models/exploration_query_result.clj
+++ b/src/metabase/explorations/models/exploration_query_result.clj
@@ -1,16 +1,14 @@
(ns metabase.explorations.models.exploration-query-result
(:require
- [clojure.edn :as edn]
[metabase.api.common :as api]
[metabase.explorations.composite :as composite]
[metabase.explorations.db :as explorations.db]
+ [metabase.interestingness.core :as interestingness]
[metabase.models.interface :as mi]
[metabase.queries.core :as queries]
[metabase.query-permissions.core :as query-perms]
[metabase.query-processor.core :as qp]
- [metabase.util.encryption :as encryption]
[metabase.util.i18n :refer [tru]]
- [metabase.util.log :as log]
[methodical.core :as methodical]
[toucan2.core :as t2])
(:import
@@ -24,41 +22,14 @@
(doto :model/ExplorationQueryResult
(derive :metabase/model))
-(defn- edn-in
- "Encode a value as EDN. JSON would mangle `compute-chart-stats` output: `:chart-type` is a keyword,
- `:series` is keyed by series-name *strings*, and the histogram `:distribution
- :estimated-percentiles` is keyed by *integers* — only EDN round-trips all three."
- [v]
- (cond
- (nil? v) nil
- (string? v) v
- :else (pr-str v)))
-
-(defn- edn-out
- "Decode an EDN blob, recovering `nil` (with a warning) on parse failure rather than crashing the
- whole `t2/select`. Bad rows can come from data written under an earlier transform (e.g. JSON), or
- from forward-compat scenarios where the writer used types the reader can't parse — neither should
- ever break a read."
- [s]
- (when (string? s)
- (try
- (edn/read-string {:readers {} :default (fn [tag v] [::unknown-tag tag v])} s)
- (catch Throwable e
- (log/warn e "Failed to parse an exploration_query_result EDN column; returning nil")
- nil))))
-
-(def ^:private transform-encrypted-edn
- "[[metabase.models.interface/transform-encrypted-text]] over a value serialized as EDN."
- {:in (comp encryption/maybe-encrypt edn-in)
- :out (comp edn-out (mi/decrypt-error-context "exploration_query_result.chart_stats" encryption/maybe-decrypt))})
-
;; Every column here is encrypted at rest, and for one reason: each holds warehouse values, or prose
;; derived from them, produced under the creator's data-access lens. That is the same material as the
;; row blob in `stored_result.result_data`, which is encrypted too. `chart_stats` is easy to read as
;; mere shape and is not — the categorical stats carry each top category's `:name` straight from the
;; result rows (see [[metabase.interestingness.chart.categorical]]).
(t2/deftransforms :model/ExplorationQueryResult
- {:chart_stats transform-encrypted-edn
+ {:chart_stats (mi/transform-encrypted-json-with-schema "exploration_query_result.chart_stats"
+ interestingness/chart-stats-schema)
:metric_description (mi/transform-encrypted-text "exploration_query_result.metric_description")
:chart_description (mi/transform-encrypted-text "exploration_query_result.chart_description")})
diff --git a/src/metabase/explorations/models/exploration_thread.clj b/src/metabase/explorations/models/exploration_thread.clj
index b21050f6111..16b1d07b996 100644
--- a/src/metabase/explorations/models/exploration_thread.clj
+++ b/src/metabase/explorations/models/exploration_thread.clj
@@ -1,10 +1,9 @@
(ns metabase.explorations.models.exploration-thread
(:require
- [clojure.edn :as edn]
[metabase.explorations.db :as explorations.db]
+ [metabase.explorations.query-plan.transcript :as transcript]
[metabase.models.interface :as mi]
[metabase.permissions.core :as perms]
- [metabase.util.log :as log]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -15,30 +14,9 @@
(derive :hook/timestamped?)
(derive :hook/entity-id))
-(defn- transcript-in
- "EDN-encode the `query_plan_transcript`. EDN over JSON for the same
- reason as chart_stats: the transcript contains keyword keys, prose-mirror
- doc trees with string-keyed maps, and nested Clojure data — JSON would
- mangle the shape."
- [v]
- (cond
- (nil? v) nil
- (string? v) v
- :else (pr-str v)))
-
-(defn- transcript-out
- "Decode the EDN blob, recovering nil (with a warning) on parse failure so a
- malformed transcript can't break a read of the thread."
- [s]
- (when (string? s)
- (try
- (edn/read-string {:readers {} :default (fn [tag v] [::unknown-tag tag v])} s)
- (catch Throwable e
- (log/warn e "Failed to parse exploration_thread transcript column; returning nil")
- nil))))
-
(t2/deftransforms :model/ExplorationThread
- {:query_plan_transcript {:in transcript-in :out transcript-out}
+ {:query_plan_transcript (mi/transform-json-with-schema "exploration_thread.query_plan_transcript"
+ ::transcript/transcript)
:data_access_token perms/data-access-token-transform})
(defmethod mi/can-read? :model/ExplorationThread
diff --git a/src/metabase/explorations/query_plan.clj b/src/metabase/explorations/query_plan.clj
index 17515dfc813..03f2519429a 100644
--- a/src/metabase/explorations/query_plan.clj
+++ b/src/metabase/explorations/query_plan.clj
@@ -308,11 +308,11 @@
return the outcome keyword (`:ok`, `:skip-empty`, or `:failed`)."
[{:keys [thread-id metric-by-key] :as ctx} picked planner-id pre]
(let [{:keys [outcome plan rationale transcript final-errors]} (planner/plan! picked ctx)
- transcript-body {:outcome outcome
- :rationale rationale
- :plan plan
- :final-errors final-errors
- :planner transcript}]
+ transcript-body {:outcome outcome
+ :rationale rationale
+ :plan plan
+ :final-errors final-errors
+ :planner-notes transcript}]
(case outcome
:ok
(let [result (insert-plan-rows! thread-id metric-by-key plan)]
diff --git a/src/metabase/explorations/query_plan/transcript.clj b/src/metabase/explorations/query_plan/transcript.clj
new file mode 100644
index 00000000000..ce2072ea2f4
--- /dev/null
+++ b/src/metabase/explorations/query_plan/transcript.clj
@@ -0,0 +1,35 @@
+(ns metabase.explorations.query-plan.transcript
+ "Schema for `exploration_thread.query_plan_transcript`."
+ (:require
+ [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(mr/def ::plan-item
+ "One item a planner emits — chart this metric against this dimension, in this variant."
+ [:map
+ [:block_id :int]
+ [:metric_id :int]
+ [:dimension_id :string]
+ [:variant :string]
+ [:params {:optional true} [:maybe :map]]
+ [:rationale {:optional true} [:maybe :string]]])
+
+(mr/def ::planner-transcript
+ "A planner's own account of its run, nested under the orchestrator's `:transcript`."
+ [:map
+ [:outcome {:optional true} [:maybe :keyword]]
+ [:rationale {:optional true} [:maybe :string]]
+ [:plan {:optional true} [:maybe [:sequential ::plan-item]]]
+ [:planner-notes {:optional true} [:maybe :map]]])
+
+(mr/def ::transcript
+ "One planning run, as persisted. Open: `:rows-count`, `:error` and anything else an outcome carries
+ ride through as-is."
+ [:map
+ [:generated-at {:optional true} :string]
+ [:thread-id {:optional true} :int]
+ [:planner {:optional true} [:maybe :keyword]]
+ [:outcome {:optional true} [:maybe :keyword]]
+ [:note {:optional true} [:maybe :keyword]]
+ [:transcript {:optional true} [:maybe ::planner-transcript]]])
diff --git a/src/metabase/interestingness/chart.clj b/src/metabase/interestingness/chart.clj
index bb13eb68f67..ad2d40b4855 100644
--- a/src/metabase/interestingness/chart.clj
+++ b/src/metabase/interestingness/chart.clj
@@ -69,7 +69,7 @@
[{:keys [series correlations], :as _stats}]
(if (empty? series)
0.0
- (let [per-series (for [[_ s] series]
+ (let [per-series (for [s series]
(let [has-trend (some-> (:trend s) trend-interesting?)
has-outlier (seq (:outliers s))
has-change (seq (:significant-changes s))
@@ -88,7 +88,7 @@
[{:keys [series]}]
(if (empty? series)
0.0
- (let [scores (for [[_ s] series]
+ (let [scores (for [s series]
(let [cc (:category-count s 0)
has-outlier (seq (:outliers s))
range-ratio (when-let [{:keys [min max]} (:summary s)]
@@ -106,7 +106,7 @@
[{:keys [series]}]
(if (empty? series)
0.0
- (let [scores (for [[_ s] series]
+ (let [scores (for [s series]
(let [coef (some-> s :correlation :coefficient double abs)
r2 (some-> s :regression :r-squared double)
has-outlier (seq (:outliers s))]
@@ -121,7 +121,7 @@
[{:keys [series]}]
(if (empty? series)
0.0
- (let [scores (for [[_ s] series]
+ (let [scores (for [s series]
(let [{:keys [peak-count concentration-top3]} (:structure s)
skew (some-> s :distribution :weighted-skewness double abs)]
(cond-> 0.0
diff --git a/src/metabase/interestingness/chart/histogram.clj b/src/metabase/interestingness/chart/histogram.clj
index c9cffbbe0ec..e3e06e18726 100644
--- a/src/metabase/interestingness/chart/histogram.clj
+++ b/src/metabase/interestingness/chart/histogram.clj
@@ -69,13 +69,17 @@
(+ x-prev (* t (- x-curr x-prev)))))
(recur (inc i))))))))
+(def ^:private estimated-percentiles
+ "The percentiles [[compute-estimated-percentiles]] estimates, and the field each lands under."
+ [[:p25 25] [:p50 50] [:p75 75] [:p90 90] [:p95 95] [:p99 99]])
+
(defn- compute-estimated-percentiles
"Compute estimated percentiles from binned data using cumulative interpolation."
[sorted-xs cum-counts total]
- (let [pcts [25 50 75 90 95 99]]
- (zipmap pcts
- (mapv #(cumulative-percentile sorted-xs cum-counts total (/ (double %) 100.0))
- pcts))))
+ (into {}
+ (map (fn [[field pct]]
+ [field (cumulative-percentile sorted-xs cum-counts total (/ (double pct) 100.0))]))
+ estimated-percentiles))
;;; -------------------------------------------- Structural Metrics ---------------------------------------------------
@@ -126,7 +130,7 @@
:total-count 0
:data-points n
:bin-data []
- :distribution {:estimated-percentiles {}
+ :distribution {:estimated-percentiles nil
:estimated-quartiles {:q1 0 :median 0 :q3 0 :iqr 0}}
:structure {:mode-bin nil
:peak-count 0
@@ -147,9 +151,9 @@
max-x (last sorted-xs)
;; Distribution estimates
percentiles (compute-estimated-percentiles sorted-xs cum-counts total)
- q1 (get percentiles 25)
- med (get percentiles 50)
- q3 (get percentiles 75)
+ q1 (:p25 percentiles)
+ med (:p50 percentiles)
+ q3 (:p75 percentiles)
;; Shape metrics (weighted)
wskew (when (>= n min-shape-metrics-points)
(stats.u/nan->nil (or (weighted-skewness sorted-xs sorted-ys total wmean wstd) ##NaN)))
diff --git a/src/metabase/interestingness/chart/repr.clj b/src/metabase/interestingness/chart/repr.clj
index 63af2536874..da417e5ae38 100644
--- a/src/metabase/interestingness/chart/repr.clj
+++ b/src/metabase/interestingness/chart/repr.clj
@@ -69,26 +69,18 @@
:else (str d)))
(defn- trend-direction-text
- "Convert trend direction keyword to readable text."
+ "Convert a trend direction keyword to readable text."
[direction]
(case direction
:strongly-increasing "strongly increasing"
- :increasing "increasing"
- :flat "flat"
- :decreasing "decreasing"
:strongly-decreasing "strongly decreasing"
- :no-clear-trend "no clear trend"
+ :no-clear-trend "no clear trend"
(name direction)))
(defn- volatility-level-text
- "Convert volatility level keyword to readable text."
+ "Convert a volatility level keyword to readable text."
[level]
- (case level
- :low "low"
- :moderate "moderate"
- :high "high"
- :extreme "extreme"
- (name level)))
+ (name level))
(defn- render-axis-range
"Render '**<label> Range**: min to max' from a summary map with :min/:max."
@@ -115,7 +107,7 @@
[{:keys [downsampled-series correlations-capped]}]
(let [parts (cond-> []
(seq downsampled-series)
- (conj (let [entries (for [[name {:keys [original-count sampled-count]}] downsampled-series]
+ (conj (let [entries (for [{:keys [name original-count sampled-count]} downsampled-series]
(str name " (" original-count " → " sampled-count " points)"))]
(str "Data was downsampled for statistical analysis: "
(str/join ", " entries)
@@ -376,10 +368,10 @@
", range=" (format-number data-range))
p-str (when (seq estimated-percentiles)
(str "**Estimated Percentiles**: "
- "P25≈" (format-number (get estimated-percentiles 25))
- ", P50≈" (format-number (get estimated-percentiles 50))
- ", P75≈" (format-number (get estimated-percentiles 75))
- ", P90≈" (format-number (get estimated-percentiles 90))))
+ "P25≈" (format-number (:p25 estimated-percentiles))
+ ", P50≈" (format-number (:p50 estimated-percentiles))
+ ", P75≈" (format-number (:p75 estimated-percentiles))
+ ", P90≈" (format-number (:p90 estimated-percentiles))))
iqr-str (when estimated-quartiles
(str "**Estimated IQR**: " (format-number (:iqr estimated-quartiles))
" (Q1≈" (format-number (:q1 estimated-quartiles))
@@ -422,8 +414,8 @@
"**Series Count**: " series-count)
limits-note (when limits (render-limits-note limits))
series-sections (str/join "\n\n"
- (for [[sname s] series]
- (render-series-fn sname s)))]
+ (for [s series]
+ (render-series-fn (:name s) s)))]
(str/join "\n\n"
(remove str/blank?
(concat [header limits-note]
diff --git a/src/metabase/interestingness/chart/stats.clj b/src/metabase/interestingness/chart/stats.clj
index 7b29972f04b..b19f3db8b83 100644
--- a/src/metabase/interestingness/chart/stats.clj
+++ b/src/metabase/interestingness/chart/stats.clj
@@ -48,10 +48,10 @@
[name (count y_values)]))
limited-series (into {} (for [[name config] series-data]
[name (downsample-series config max-data-points-per-series)]))
- downsampled (into {} (for [[name orig-count] original-counts
- :let [new-count (count (get-in limited-series [name :y_values]))]
- :when (< new-count orig-count)]
- [name {:original-count orig-count :sampled-count new-count}]))]
+ downsampled (vec (for [[name orig-count] original-counts
+ :let [new-count (count (get-in limited-series [name :y_values]))]
+ :when (< new-count orig-count)]
+ {:name name :original-count orig-count :sampled-count new-count}))]
[limited-series
(when (seq downsampled)
{:downsampled-series downsampled})]))
@@ -168,3 +168,9 @@
" charts not yet implemented")})]
(cond-> stats
limits-info (assoc :limits limits-info))))
+
+;;; --------------------------------------------- Serialization ------------------------------------------------------
+
+(def chart-stats-schema
+ "Registry key for a [[compute-chart-stats]] result."
+ ::stats.types/chart-stats)
diff --git a/src/metabase/interestingness/chart/time_series.clj b/src/metabase/interestingness/chart/time_series.clj
index 3bacace7fb1..623352a2c2e 100644
--- a/src/metabase/interestingness/chart/time_series.clj
+++ b/src/metabase/interestingness/chart/time_series.clj
@@ -51,7 +51,7 @@
and can flatly contradict each other — the source of labels like
'strongly increasing (-97.9% overall)'. When the slope direction disagrees in
*sign* with the change the reader actually sees, or when the series is so noisy
- that its swings dwarf its mean (CV ≥ 1.0), report `:no-clear-trend` rather than
+ that its swings dwarf its mean (CV >= 1.0), report `:no-clear-trend` rather than
asserting a direction.
Note we deliberately do NOT suppress on moderate CV: a clean, wide ramp
@@ -81,7 +81,7 @@
end-val (last values-vec)
mean-val (dfn/mean values)
std-dev (dfn/standard-deviation values)
- cv (if (zero? mean-val) 0.0 (/ std-dev (Math/abs (double mean-val))))
+ cv (or (stats.u/nan->nil (if (zero? mean-val) 0.0 (/ std-dev (Math/abs (double mean-val))))) 0.0)
change-pct (stats.u/percentage-change start-val end-val)]
{:direction (reconcile-direction (slope-to-direction slope mean-val n) change-pct cv)
:overall-change-pct change-pct
@@ -109,7 +109,7 @@
[values]
(let [mean-val (dfn/mean values)
std-dev (dfn/standard-deviation values)
- cv (if (zero? mean-val) 0.0 (/ std-dev (Math/abs (double mean-val))))
+ cv (or (stats.u/nan->nil (if (zero? mean-val) 0.0 (/ std-dev (Math/abs (double mean-val))))) 0.0)
values-vec (vec values)
pct-changes (for [i (range 1 (count values-vec))
:let [prev (nth values-vec (dec i))
diff --git a/src/metabase/interestingness/chart/types.clj b/src/metabase/interestingness/chart/types.clj
index 1d10ab741e9..99a7c45f8c1 100644
--- a/src/metabase/interestingness/chart/types.clj
+++ b/src/metabase/interestingness/chart/types.clj
@@ -58,7 +58,7 @@
[:max number?]
[:mean number?]
[:median number?]
- [:std-dev number?]
+ [:std-dev [:maybe number?]]
[:range number?]])
(mr/def ::time-range
@@ -159,9 +159,10 @@
(mr/def ::stats-limits
"Notes about data-volume limits applied before computing chart statistics."
[:map {:closed true}
- [:downsampled-series {:optional true} [:map-of :string [:map {:closed true}
- [:original-count :int]
- [:sampled-count :int]]]]
+ [:downsampled-series {:optional true} [:sequential [:map {:closed true}
+ [:name :string]
+ [:original-count :int]
+ [:sampled-count :int]]]]
[:correlations-capped {:optional true} [:map {:closed true}
[:total-series :int]
[:max-correlated :int]]]])
@@ -177,6 +178,7 @@
(mr/def ::time-series-series-stats
"Statistics for a single time series."
[:map {:closed true}
+ [:name :string]
[:summary ::series-summary]
[:time-range ::time-range]
[:data-points :int]
@@ -198,7 +200,7 @@
[:map {:closed true}
[:chart-type [:= :time-series]]
[:series-count :int]
- [:series [:map-of :string ::time-series-series-stats]]
+ [:series [:sequential ::time-series-series-stats]]
[:correlations {:optional true} [:maybe [:sequential ::correlation]]]
[:limits {:optional true} ::stats-limits]])
@@ -212,6 +214,7 @@
(mr/def ::categorical-series-stats
"Statistics for a single categorical series."
[:map {:closed true}
+ [:name :string]
[:summary [:maybe ::series-summary]]
[:data-points :int]
[:category-count :int]
@@ -226,7 +229,7 @@
[:map {:closed true}
[:chart-type [:= :categorical]]
[:series-count :int]
- [:series [:map-of :string ::categorical-series-stats]]
+ [:series [:sequential ::categorical-series-stats]]
[:correlations {:optional true} [:maybe [:sequential ::correlation]]]
[:limits {:optional true} ::stats-limits]])
@@ -240,6 +243,7 @@
(mr/def ::scatter-series-stats
"Statistics for a single scatter series."
[:map {:closed true}
+ [:name :string]
[:x-summary [:maybe ::series-summary]]
[:y-summary [:maybe ::series-summary]]
[:data-points :int]
@@ -258,7 +262,7 @@
[:map {:closed true}
[:chart-type [:= :scatter]]
[:series-count :int]
- [:series [:map-of :string ::scatter-series-stats]]
+ [:series [:sequential ::scatter-series-stats]]
[:limits {:optional true} ::stats-limits]])
(mr/def ::histogram-summary
@@ -268,10 +272,20 @@
[:weighted-std-dev number?]
[:data-range number?]])
+(mr/def ::estimated-percentiles
+ "The fixed percentile set [[metabase.interestingness.chart.histogram]] estimates."
+ [:map {:closed true}
+ [:p25 number?]
+ [:p50 number?]
+ [:p75 number?]
+ [:p90 number?]
+ [:p95 number?]
+ [:p99 number?]])
+
(mr/def ::estimated-distribution-stats
"Distribution statistics estimated from binned histogram data using weighted approximations."
[:map {:closed true}
- [:estimated-percentiles [:map-of :int number?]]
+ [:estimated-percentiles [:maybe ::estimated-percentiles]]
[:estimated-quartiles [:map {:closed true}
[:q1 number?]
[:median number?]
@@ -293,6 +307,7 @@
(mr/def ::histogram-series-stats
"Statistics for a single histogram series."
[:map {:closed true}
+ [:name :string]
[:estimated-summary ::histogram-summary]
[:total-count :int]
[:data-points :int]
@@ -307,7 +322,7 @@
[:map {:closed true}
[:chart-type [:= :histogram]]
[:series-count :int]
- [:series [:map-of :string ::histogram-series-stats]]
+ [:series [:sequential ::histogram-series-stats]]
[:limits {:optional true} ::stats-limits]])
(mr/def ::unknown-stats
@@ -320,7 +335,7 @@
(mr/def ::chart-stats
"Union of all chart statistics types, dispatched on `:chart-type`."
- [:multi {:dispatch :chart-type}
+ [:multi {:dispatch (comp keyword :chart-type)}
[:time-series ::time-series-stats]
[:categorical ::categorical-stats]
[:scatter ::scatter-stats]
diff --git a/src/metabase/interestingness/chart/util.clj b/src/metabase/interestingness/chart/util.clj
index 4c331e0a170..b71f1e572c5 100644
--- a/src/metabase/interestingness/chart/util.clj
+++ b/src/metabase/interestingness/chart/util.clj
@@ -8,9 +8,9 @@
(set! *warn-on-reflection* true)
(mu/defn nan->nil :- [:maybe number?]
- "Convert NaN to nil, pass through other values."
+ "Convert JSON-unrepresentable NaN and the infinities to nil. Pass every other value through."
[x :- number?]
- (when-not (Double/isNaN (double x))
+ (when (Double/isFinite (double x))
x))
(mu/defn compute-summary :- ::stats.types/series-summary
@@ -22,7 +22,7 @@
:max max-val
:mean (dfn/mean values)
:median (dfn/median values)
- :std-dev (dfn/standard-deviation values)
+ :std-dev (nan->nil (dfn/standard-deviation values))
:range (- max-val min-val)}))
(mu/defn correlation-direction :- ::stats.types/correlation-direction
@@ -41,7 +41,7 @@
(>= abs-coef 0.7) :strong
(>= abs-coef 0.4) :moderate
(>= abs-coef 0.2) :weak
- :else :none)))
+ :else :none)))
(defn- align-series-on-x
"Align two series on common x-values using listwise deletion.
@@ -106,24 +106,26 @@
(* 100.0 (/ (- to-val from-val) (Math/abs (double from-val))))))
(defn compute-series-with-labels
- "Apply `compute-fn` to each series' x_values and y_values, attaching :x-name and :y-name
- from column metadata. `compute-fn` is called on x_values and y_values for each series.
- Returns a map of series-name -> stats-with-labels."
+ "Apply `compute-fn` to each series' x_values and y_values, attaching `:name` plus `:x-name` and
+ `:y-name` from column metadata. `compute-fn` is called on x_values and y_values for each series.
+
+ Returns a *vector* of series-stats maps, each carrying its own `:name`."
[series-data compute-fn]
- (into {}
- (for [[series-name {:keys [x_values y_values x y]}] series-data]
- [series-name (-> (compute-fn x_values y_values)
- (assoc :x-name (some-> x :name))
- (assoc :y-name (some-> y :name)))])))
+ (mapv (fn [[series-name {:keys [x_values y_values x y]}]]
+ (-> (compute-fn x_values y_values)
+ (assoc :name series-name)
+ (assoc :x-name (some-> x :name))
+ (assoc :y-name (some-> y :name))))
+ series-data))
(mu/defn make-chart-result :- ::stats.types/chart-stats
"Build the standard chart stats result map."
[chart-type :- [:enum :time-series :categorical :scatter :histogram :unknown]
series-data :- [:map-of :string ::stats.types/series-config]
- series-stats :- [:map-of :string [:or ::stats.types/time-series-series-stats
- ::stats.types/categorical-series-stats
- ::stats.types/scatter-series-stats
- ::stats.types/histogram-series-stats]]
+ series-stats :- [:sequential [:or ::stats.types/time-series-series-stats
+ ::stats.types/categorical-series-stats
+ ::stats.types/scatter-series-stats
+ ::stats.types/histogram-series-stats]]
correlations :- [:maybe [:sequential ::stats.types/correlation]]]
(cond-> {:chart-type chart-type
:series-count (count series-data)
diff --git a/src/metabase/interestingness/core.clj b/src/metabase/interestingness/core.clj
index 58b68ff4df2..c56b8a7e32d 100644
--- a/src/metabase/interestingness/core.clj
+++ b/src/metabase/interestingness/core.clj
@@ -25,4 +25,5 @@
generate-representation
temporal-context]
[metabase.interestingness.chart.stats
+ chart-stats-schema
compute-chart-stats])
diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.clj
index 859ef737505..ed707e729a6 100644
--- a/src/metabase/models/interface.clj
+++ b/src/metabase/models/interface.clj
@@ -14,6 +14,8 @@
[clojure.spec.alpha :as s]
[clojure.string :as str]
[clojure.walk :as walk]
+ [malli.core :as mc]
+ [malli.transform :as mtx]
[medley.core :as m]
[metabase.app-db.core :as mdb]
;; Toucan out-transforms normalize stored legacy MBQL on read; needed until the app db is MBQL 5
@@ -349,6 +351,59 @@
{:in encrypted-json-in
:out (decrypt-error-context source cached-encrypted-json-out)})
+(def ^:private json-schema-transformer
+ "The same [[malli.transform/json-transformer]] [[metabase.api.macros]] decodes request bodies with, so a
+ value has one JSON shape rather than one for the wire and another for the app DB."
+ (mtx/json-transformer))
+
+(defn- json-schema-encoder [schema]
+ (mr/cached ::json-schema-encoder schema #(mc/encoder schema json-schema-transformer)))
+
+(defn- json-schema-decoder [schema]
+ (mr/cached ::json-schema-decoder schema #(mc/decoder schema json-schema-transformer)))
+
+(defn transform-json-with-schema
+ "Like [[transform-json]], but runs the value through `schema`'s malli JSON encoder and decoder, so a column
+ can hold keywords and anything else JSON cannot carry on its own. `source` is a \"table.column\" string,
+ named in the log message when a blob cannot be read.
+
+ Values always go through `schema`; there is no string passthrough, since an already-encoded string would
+ skip the decode this transform exists to do.
+
+ An unreadable blob reads as `nil` rather than throwing, so one bad row cannot break a whole `t2/select`. It
+ is logged at `:log-level` (`:warn` by default); pass `:error` when a failure carries a security
+ consequence."
+ ([source schema]
+ (transform-json-with-schema source schema nil))
+
+ ([source schema {:keys [log-level] :or {log-level :warn}}]
+ {:in (fn [v]
+ (when (some? v)
+ (json/encode ((json-schema-encoder schema) v))))
+ :out (fn [s]
+ (when (string? s)
+ (try
+ ((json-schema-decoder schema) (json/decode+kw s))
+ (catch Throwable e
+ (if (= log-level :error)
+ (log/errorf e "Failed to parse %s; returning nil" source)
+ (log/warnf e "Failed to parse %s; returning nil" source))
+ nil))))}))
+
+(defn transform-encrypted-json-with-schema
+ "[[transform-json-with-schema]] for a column that is also encrypted at rest. [[decrypt-error-context]] names
+ the column when a decrypt fails, instead of a bare \"Expected an encrypted value\".
+
+ Unlike [[transform-encrypted-json]], the decrypt is not memoized: that cache holds both ciphertext and
+ decoded value for an hour, which is the wrong trade for the unbounded blobs these columns tend to hold."
+ ([source schema]
+ (transform-encrypted-json-with-schema source schema nil))
+
+ ([source schema opts]
+ (let [{:keys [in out]} (transform-json-with-schema source schema opts)]
+ {:in (comp encryption/maybe-encrypt in)
+ :out (comp out (decrypt-error-context source encryption/maybe-decrypt))})))
+
(defn transform-encrypted-text
"Whole-column encrypted text transform for the column named by `source` (a \"table.column\" string, used in decrypt
error messages). When `MB_ENCRYPTION_SECRET_KEY` is set, a plaintext value at rest is rejected on read (see
diff --git a/src/metabase/permissions/data_access_token.clj b/src/metabase/permissions/data_access_token.clj
index 28f103f41c0..d9ec5ab4baa 100644
--- a/src/metabase/permissions/data_access_token.clj
+++ b/src/metabase/permissions/data_access_token.clj
@@ -22,7 +22,7 @@
unavailable — fail closed, never leak.
Each contributor's raw value is replaced by a [[digest]] of it before it leaves this namespace.
- A token is persisted verbatim (`stored_result.data_access_token`, plaintext EDN) and the raw
+ A token is persisted verbatim (`stored_result.data_access_token`, plaintext JSON) and the raw
values can contain sensitive information for sandbox contributors and others.
Nothing here needs to read those values back so a one-way digest keeps the
gate's semantics exactly while keeping the values out of a table whose rows outlive the attributes
@@ -34,9 +34,9 @@
(:require
[buddy.core.codecs :as codecs]
[buddy.core.hash :as buddy-hash]
- [clojure.edn :as edn]
+ [metabase.models.interface :as mi]
[metabase.premium-features.core :refer [defenterprise]]
- [metabase.util.log :as log]))
+ [metabase.util.malli.registry :as mr]))
(set! *warn-on-reflection* true)
@@ -119,29 +119,20 @@
[creator-token viewer-token]
(= creator-token viewer-token))
-(defn- token-in
- "Serialize a token as EDN. JSON can't be used: the token is keyed by integer table-id /
- database-id, and JSON mangles non-string map keys. `nil` is stored as SQL NULL rather than the
- string \"nil\"."
- [v]
- (when (some? v)
- (pr-str v)))
-
-(defn- token-out
- "Read a token back. Reader tags are refused rather than dispatched — this parses a column, and
- nothing legitimately writes a tagged literal into one. An unreadable blob decodes to `nil`, which
- every gate built on [[data-access-compatible?]] denies to non-superusers: fail closed, never widen
- access on a parse error. Logged at ERROR because a write path that persisted an unreadable token
- is a bug, and the denial it causes is otherwise hard to trace."
- [s]
- (when (string? s)
- (try
- (edn/read-string {:readers {} :default (fn [_tag v] v)} s)
- (catch Throwable e
- (log/error e "Failed to parse a stored data_access_token; the read gate will deny non-admins")
- nil))))
+(mr/def ::token
+ "A persisted [[data-access-token]]. Declared so the transform can put the integer target ids back
+ after a JSON round trip: [[data-access-compatible?]] is bare `=`, and `{:10 …}` does not equal
+ `{10 …}`."
+ [:map
+ [:sandbox {:optional true} [:map-of :int :string]]
+ [:impersonation {:optional true} [:map-of :int :string]]
+ [:routing {:optional true} [:map-of :int :string]]])
(def data-access-token-transform
"Toucan transform for a persisted [[data-access-token]]. Used by every table that stamps the lens
- its content was produced under."
- {:in token-in :out token-out})
+ its content was produced under.
+
+ Logged at ERROR rather than the default WARN: a write path that persisted an unreadable token is a
+ bug, and the denial it causes -- an unreadable blob reads as `nil`, which every gate built on
+ [[data-access-compatible?]] denies to non-superusers -- is otherwise hard to trace."
+ (mi/transform-json-with-schema "stored_result.data_access_token" ::token {:log-level :error}))
diff --git a/test/metabase/explorations/models/exploration_query_result_test.clj b/test/metabase/explorations/models/exploration_query_result_test.clj
index 3c578ebfb30..a37f3bfd61e 100644
--- a/test/metabase/explorations/models/exploration_query_result_test.clj
+++ b/test/metabase/explorations/models/exploration_query_result_test.clj
@@ -7,6 +7,7 @@
[metabase.lib.metadata :as lib.metadata]
[metabase.test :as mt]
[metabase.util.encryption-test :as encryption-test]
+ [metabase.util.json :as json]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -22,11 +23,45 @@
(def ^:private stats-with-warehouse-values
"The shape `compute-chart-stats` produces for a categorical chart: each top category's `:name`
- comes straight from the result rows, so this column holds verbatim warehouse values."
+ comes straight from the result rows, so this column holds verbatim warehouse values. Every value
+ in it is a plain JSON value except the `:chart-type` tag, which the schema-driven codec restores."
{:chart-type :categorical
- :series {"Count" {:top-categories [{:name "ACME Corp" :value 41}
- {:name "Initech" :value 12}]
- :category-count 2}}})
+ :series [{:name "Count"
+ :top-categories [{:name "ACME Corp" :value 41}
+ {:name "Initech" :value 12}]
+ :category-count 2}]})
+
+(def ^:private time-series-stats
+ "Every keyword-valued field a time-series result carries. JSON has no keywords, so these are what
+ the `mc/encode`/`mc/decode` pair over `::chart-stats` exists to preserve — the storage format does
+ not get to decide that the data model uses strings. The series name holds a `/`, which rides
+ through as a value rather than as a map key."
+ {:chart-type :time-series
+ :series-count 1
+ :series [{:name "Revenue / Cost"
+ :summary {:min 1.0 :max 9.0 :mean 5.0 :median 5.0 :std-dev 2.5 :range 8.0}
+ :trend {:direction :strongly-increasing :overall-change-pct 12.5}
+ :volatility {:level :moderate :coefficient-of-variation 0.3}
+ :patterns [{:type :spike :description "a spike in March"}]
+ :data-points 7}]
+ :correlations [{:series-a "Revenue / Cost" :series-b "Units"
+ :coefficient 0.91 :strength :strong :direction :positive}]})
+
+(def ^:private histogram-stats
+ "A histogram's estimated percentiles, under the named fields the producer computes."
+ {:chart-type :histogram
+ :series-count 1
+ :series [{:name "Total"
+ :total-count 120
+ :distribution {:estimated-percentiles {:p25 1.5 :p50 2.5 :p75 5.0 :p90 9.0 :p95 9.5 :p99 9.9}
+ :estimated-quartiles {:q1 1.5 :median 2.5 :q3 9.0 :iqr 7.5}}}]})
+
+(def ^:private single-point-stats
+ "A single-point series has no standard deviation. `util/nan->nil` drops the `##NaN` at the source,
+ so nothing non-finite — which JSON cannot represent — ever reaches the column."
+ {:chart-type :categorical
+ :series [{:name "Count"
+ :summary {:min 5.0 :max 5.0 :mean 5.0 :median 5.0 :std-dev nil :range 0.0}}]})
(defn- query-result-row!
"Insert one `exploration_query_result` carrying `stats`, returning its id."
@@ -59,11 +94,33 @@
:chart_stats stats
:metric_description "a description"})))
+(defn- raw-chart-stats
+ [id]
+ (:chart_stats (t2/query-one {:select [:chart_stats]
+ :from [:exploration_query_result]
+ :where [:= :id id]})))
+
+(deftest chart-stats-is-stored-as-json-test
+ (testing "chart_stats is plain JSON at rest — the keywords the stats carry in memory are written
+ out as strings, and nothing keyword-shaped survives into the stored bytes"
+ (encryption-test/with-secret-key nil
+ (let [id (query-result-row! stats-with-warehouse-values)]
+ (is (= {"chart-type" "categorical"
+ "series" [{"name" "Count"
+ "top-categories" [{"name" "ACME Corp" "value" 41}
+ {"name" "Initech" "value" 12}]
+ "category-count" 2}]}
+ (json/decode (raw-chart-stats id))))))))
+
(deftest chart-stats-round-trip-test
- (testing "chart_stats survives the EDN transform with its keyword and string-keyed shape intact"
- (let [id (query-result-row! stats-with-warehouse-values)]
- (is (= stats-with-warehouse-values
- (t2/select-one-fn :chart_stats :model/ExplorationQueryResult :id id))))))
+ (testing "chart_stats survives the round trip with its keywords intact. The stored bytes hold
+ strings (see above); the schema-driven codec is what puts `:chart-type`, a trend's
+ `:direction`, a volatility `:level`, a pattern's `:type` and a correlation's
+ `:strength`/`:direction` back on the way out"
+ (doseq [stats [stats-with-warehouse-values time-series-stats histogram-stats single-point-stats]]
+ (let [id (query-result-row! stats)]
+ (is (= stats (t2/select-one-fn :chart_stats :model/ExplorationQueryResult :id id))
+ (pr-str (:chart-type stats)))))))
(deftest chart-stats-is-encrypted-at-rest-test
;; isolated app DB: runs with an encryption key active, so nothing here may touch the shared test DB
@@ -74,9 +131,7 @@
It must not sit in the clear next to them."
(encryption-test/with-secret-key "chart-stats-encryption-test-key"
(let [id (query-result-row! stats-with-warehouse-values)
- raw (:chart_stats (t2/query-one {:select [:chart_stats]
- :from [:exploration_query_result]
- :where [:= :id id]}))]
+ raw (raw-chart-stats id)]
(testing "the stored bytes do not contain the warehouse value"
(is (string? raw))
(is (not (str/includes? raw "ACME Corp"))
@@ -95,7 +150,7 @@
(is (= stats-with-warehouse-values
(t2/select-one-fn :chart_stats :model/ExplorationQueryResult :id id)))
(t2/query-one {:update :exploration_query_result
- :set {:chart_stats (pr-str stats-with-warehouse-values)}
+ :set {:chart_stats (json/encode stats-with-warehouse-values)}
:where [:= :id id]})
(is (thrown? clojure.lang.ExceptionInfo
(t2/select-one-fn :chart_stats :model/ExplorationQueryResult :id id))))))))
@@ -108,7 +163,19 @@
(encryption-test/with-secret-key nil
(let [id (query-result-row! nil)]
(t2/query-one {:update :exploration_query_result
- :set {:chart_stats (pr-str stats-with-warehouse-values)}
+ :set {:chart_stats (json/encode stats-with-warehouse-values)}
:where [:= :id id]})
(is (= stats-with-warehouse-values
(t2/select-one-fn :chart_stats :model/ExplorationQueryResult :id id))))))))
+
+(deftest chart-stats-unparseable-reads-as-nil-test
+ ;; isolated app DB: writes a deliberately malformed value, so nothing here may touch the shared test DB
+ (mt/with-temp-empty-app-db [_conn :h2]
+ (mdb/setup-db! :create-sample-content? false)
+ (testing "an unparseable blob is logged and read as nil rather than breaking the whole select"
+ (encryption-test/with-secret-key nil
+ (let [id (query-result-row! nil)]
+ (t2/query-one {:update :exploration_query_result
+ :set {:chart_stats "{not json ]["}
+ :where [:= :id id]})
+ (is (nil? (t2/select-one-fn :chart_stats :model/ExplorationQueryResult :id id))))))))
diff --git a/test/metabase/explorations/models/exploration_thread_test.clj b/test/metabase/explorations/models/exploration_thread_test.clj
new file mode 100644
index 00000000000..fb136da76f3
--- /dev/null
+++ b/test/metabase/explorations/models/exploration_thread_test.clj
@@ -0,0 +1,80 @@
+(ns metabase.explorations.models.exploration-thread-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.test :as mt]
+ [metabase.util.json :as json]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private transcript
+ "What [[metabase.explorations.query-plan/record-outcome!]] persists. `:outcome`, `:note` and
+ `:planner` are keywords — the orchestrator speaks the same ones the planner protocol does, and
+ [[metabase.explorations.query-plan.transcript]] translates them at the storage boundary."
+ {:generated-at "2026-08-28T00:00:00Z"
+ :thread-id 7
+ :planner :mechanical
+ :outcome :skip-empty
+ :note :no-rows-materialized
+ :transcript {:outcome :ok
+ :plan [{:block_id 1 :metric_id 2 :dimension_id "d1" :variant "top-n-other"}]
+ :planner-notes {:strategy "mechanical" :n-items 1 :n-blocks 1}}})
+
+(def ^:private transcript-with-full-plan-item
+ "A plan item carrying every field `::transcript/plan-item` names, including the optional `:params`
+ and `:rationale` an LLM planner adds."
+ (assoc-in transcript [:transcript :plan]
+ [{:block_id 1 :metric_id 2 :dimension_id "d1" :variant "top-n-other"
+ :params {:k 5} :rationale "the tail is long"}]))
+
+(defn- thread-with-transcript!
+ [transcript]
+ (let [expl (t2/insert-returning-pk! :model/Exploration {:name "t" :creator_id (mt/user->id :lucky)})]
+ (t2/insert-returning-pk! :model/ExplorationThread {:exploration_id expl
+ :position 0
+ :query_plan_transcript transcript})))
+
+(defn- raw-transcript
+ [id]
+ (:query_plan_transcript (t2/query-one {:select [:query_plan_transcript]
+ :from [:exploration_thread]
+ :where [:= :id id]})))
+
+(deftest transcript-is-stored-as-json-test
+ (testing "query_plan_transcript is plain JSON at rest — the keywords it carries in memory are
+ written out as strings, and a plan item's `:variant`, which is a string to begin with,
+ stays one"
+ (let [id (thread-with-transcript! transcript)]
+ (is (= {"generated-at" "2026-08-28T00:00:00Z"
+ "thread-id" 7
+ "planner" "mechanical"
+ "outcome" "skip-empty"
+ "note" "no-rows-materialized"
+ "transcript" {"outcome" "ok"
+ "plan" [{"block_id" 1 "metric_id" 2
+ "dimension_id" "d1" "variant" "top-n-other"}]
+ "planner-notes" {"strategy" "mechanical" "n-items" 1 "n-blocks" 1}}}
+ (json/decode (raw-transcript id)))))))
+
+(deftest transcript-round-trip-test
+ (testing "the transcript reads back exactly as written: the schema-driven codec restores
+ `:outcome`, `:note` and `:planner` as keywords, and leaves `:variant` a string"
+ (let [id (thread-with-transcript! transcript)
+ out (t2/select-one-fn :query_plan_transcript :model/ExplorationThread :id id)]
+ (is (= transcript out))
+ (is (= :skip-empty (:outcome out)))
+ (is (= "top-n-other" (-> out :transcript :plan first :variant))))))
+
+(deftest transcript-unparseable-reads-as-nil-test
+ (testing "an unparseable blob is logged and read as nil rather than breaking the thread read"
+ (let [id (thread-with-transcript! nil)]
+ (t2/query {:update :exploration_thread
+ :set {:query_plan_transcript "{not json ]["}
+ :where [:= :id id]})
+ (is (nil? (t2/select-one-fn :query_plan_transcript :model/ExplorationThread :id id))))))
+
+(deftest transcript-plan-items-round-trip-test
+ (testing "a plan item's optional `:params` and `:rationale` survive the round trip"
+ (let [id (thread-with-transcript! transcript-with-full-plan-item)]
+ (is (= transcript-with-full-plan-item
+ (t2/select-one-fn :query_plan_transcript :model/ExplorationThread :id id))))))
diff --git a/test/metabase/interestingness/chart/categorical_test.clj b/test/metabase/interestingness/chart/categorical_test.clj
index 6eb9bd271b9..a9393dd1ec2 100644
--- a/test/metabase/interestingness/chart/categorical_test.clj
+++ b/test/metabase/interestingness/chart/categorical_test.clj
@@ -96,42 +96,42 @@
:x {:name "cat" :type "string"}
:y {:name "cost" :type "number"}
:display_name "Cost"}}]
- (is (=? {:chart-type :categorical
+ (is (=? {:chart-type :categorical
:series-count 2
- :series {"Revenue" {:summary {:min 100
- :max 200
- :mean 150.0
- :median 200.0
- :std-dev (=?/approx [70.71 0.01])
- :range 100}
- :category-count 2
- :top-categories
- [{:name "B"
- :value 200
- :percentage (=?/approx [66.66 0.01])}
- {:name "A"
- :value 100
- :percentage (=?/approx [33.33 0.01])}]
- :outliers []
- :x-name "cat"
- :y-name "rev"}
- "Cost" {:summary {:min 50
- :max 80
- :mean 65.0
- :median 80.0
- :std-dev (=?/approx [21.21 0.01])
- :range 30}
- :category-count 2
- :top-categories
- [{:name "B"
- :value 80
- :percentage (=?/approx [61.5 0.1])}
- {:name "A"
- :value 50
- :percentage (=?/approx [38.4 0.1])}]
- :outliers [],
- :x-name "cat",
- :y-name "cost"}}}
+ :series [{:name "Revenue"
+ :summary {:min 100
+ :max 200
+ :mean 150.0
+ :median 200.0
+ :std-dev (=?/approx [70.71 0.01])
+ :range 100}
+ :category-count 2
+ :top-categories [{:name "B"
+ :value 200
+ :percentage (=?/approx [66.66 0.01])}
+ {:name "A"
+ :value 100
+ :percentage (=?/approx [33.33 0.01])}]
+ :outliers []
+ :x-name "cat"
+ :y-name "rev"}
+ {:name "Cost"
+ :summary {:min 50
+ :max 80
+ :mean 65.0
+ :median 80.0
+ :std-dev (=?/approx [21.21 0.01])
+ :range 30}
+ :category-count 2
+ :top-categories [{:name "B"
+ :value 80
+ :percentage (=?/approx [61.5 0.1])}
+ {:name "A"
+ :value 50
+ :percentage (=?/approx [38.4 0.1])}]
+ :outliers []
+ :x-name "cat"
+ :y-name "cost"}]}
(categorical/compute-categorical-stats series-data {:deep? false}))))))
(deftest ^:parallel compute-categorical-stats-correlations-when-deep?-test
diff --git a/test/metabase/interestingness/chart/histogram_test.clj b/test/metabase/interestingness/chart/histogram_test.clj
index 1057aff340f..c7cfce22a0a 100644
--- a/test/metabase/interestingness/chart/histogram_test.clj
+++ b/test/metabase/interestingness/chart/histogram_test.clj
@@ -12,7 +12,7 @@
(testing "empty values returns zero counts"
(is (=? {:data-points 0
:total-count 0
- :distribution {:estimated-percentiles empty?
+ :distribution {:estimated-percentiles nil?
:estimated-quartiles {:q1 0 :median 0 :q3 0 :iqr 0}}
:structure {:mode-bin nil :peak-count 0 :bin-count 0}}
(#'histogram/compute-series-stats [])))))
@@ -21,7 +21,7 @@
(testing "all-nil values returns zero data_points"
(is (=? {:data-points 0
:total-count 0
- :distribution {:estimated-percentiles empty?
+ :distribution {:estimated-percentiles nil?
:estimated-quartiles {:q1 0 :median 0 :q3 0 :iqr 0}}}
(#'histogram/compute-series-stats [nil nil nil])))))
@@ -61,25 +61,25 @@
(testing "all six percentiles (25,50,75,90,95,99) are computed"
(let [result (#'histogram/compute-series-stats [0 10 20 30 40] [5 10 20 10 5])
pcts (get-in result [:distribution :estimated-percentiles])]
- (is (contains? pcts 25))
- (is (contains? pcts 50))
- (is (contains? pcts 75))
- (is (contains? pcts 90))
- (is (contains? pcts 95))
- (is (contains? pcts 99)))))
+ (is (contains? pcts :p25))
+ (is (contains? pcts :p50))
+ (is (contains? pcts :p75))
+ (is (contains? pcts :p90))
+ (is (contains? pcts :p95))
+ (is (contains? pcts :p99)))))
(deftest ^:parallel estimated-percentiles-ascending-test
(testing "percentile values are in ascending order"
(let [pcts (get-in (#'histogram/compute-series-stats [0 10 20 30 40] [5 10 20 10 5])
[:distribution :estimated-percentiles])]
- (is (<= (get pcts 25) (get pcts 50)))
- (is (<= (get pcts 50) (get pcts 75)))
- (is (<= (get pcts 75) (get pcts 90))))))
+ (is (<= (:p25 pcts) (:p50 pcts)))
+ (is (<= (:p50 pcts) (:p75 pcts)))
+ (is (<= (:p75 pcts) (:p90 pcts))))))
(deftest ^:parallel estimated-percentiles-symmetric-test
(testing "P50 is close to weighted mean for symmetric distribution"
(let [result (#'histogram/compute-series-stats [0 10 20 30 40] [5 10 20 10 5])
- p50 (get-in result [:distribution :estimated-percentiles 50])
+ p50 (get-in result [:distribution :estimated-percentiles :p50])
wmean (get-in result [:estimated-summary :weighted-mean])]
;; For symmetric binned data, P50 should be reasonably close to the mean
(is (< (Math/abs (double (- p50 wmean))) 10.0)))))
@@ -179,15 +179,16 @@
:display_name "Bins"}}]
(is (=? {:chart-type :histogram
:series-count 1
- :series {"Bins" {:data-points 5
- :total-count 50
- :estimated-summary {:weighted-mean (=?/approx [20.0 0.001])}
- :distribution {:estimated-quartiles {:q1 number?
- :q3 number?
- :iqr number?}}
- :structure {:mode-bin [20.0 20.0]
- :peak-count 1
- :bin-count 5}}}}
+ :series [{:name "Bins"
+ :data-points 5
+ :total-count 50
+ :estimated-summary {:weighted-mean (=?/approx [20.0 0.001])}
+ :distribution {:estimated-quartiles {:q1 number?
+ :q3 number?
+ :iqr number?}}
+ :structure {:mode-bin [20.0 20.0]
+ :peak-count 1
+ :bin-count 5}}]}
(histogram/compute-histogram-stats series-data {}))))))
(deftest ^:parallel single-arity-uses-indices-test
diff --git a/test/metabase/interestingness/chart/repr_test.clj b/test/metabase/interestingness/chart/repr_test.clj
index e6651732ee9..710cb1d87c9 100644
--- a/test/metabase/interestingness/chart/repr_test.clj
+++ b/test/metabase/interestingness/chart/repr_test.clj
@@ -67,7 +67,7 @@
series-stats (#'histogram/compute-series-stats xs ys)
stats {:chart-type :histogram
:series-count 1
- :series {"Test Series" series-stats}}
+ :series [(assoc series-stats :name "Test Series")]}
rep (repr/generate-representation {:stats stats})]
(is (str/includes? rep "Test Series"))
(is (str/includes? rep "Distribution Shape")))))
@@ -79,7 +79,7 @@
series-stats (#'histogram/compute-series-stats xs ys)
stats {:chart-type :histogram
:series-count 1
- :series {"Values" series-stats}}
+ :series [(assoc series-stats :name "Values")]}
rep (repr/generate-representation {:stats stats})]
(is (str/includes? rep "Estimated Percentiles"))
(is (str/includes? rep "P50≈"))
@@ -90,7 +90,7 @@
(let [series-stats (#'histogram/compute-series-stats [0 10 20 30 40] [5 10 20 10 5])
stats {:chart-type :histogram
:series-count 1
- :series {"Bins" series-stats}}
+ :series [(assoc series-stats :name "Bins")]}
rep (repr/generate-representation {:stats stats})]
(is (str/includes? rep "Structure"))
(is (str/includes? rep "mode bin"))
@@ -103,7 +103,7 @@
(let [series-stats (#'scatter/compute-series-stats [1 2 3 4 5] [10 20 30 40 50])
stats {:chart-type :scatter
:series-count 1
- :series {"Test Series" series-stats}}
+ :series [(assoc series-stats :name "Test Series")]}
rep (repr/generate-representation {:stats stats})]
(is (str/includes? rep "Test Series"))
(is (str/includes? rep "Relationship"))
@@ -114,7 +114,7 @@
(let [series-stats (#'scatter/compute-series-stats [1 2 3 4 5] [12 14 16 18 20])
stats {:chart-type :scatter
:series-count 1
- :series {"Linear" series-stats}}
+ :series [(assoc series-stats :name "Linear")]}
rep (repr/generate-representation {:stats stats})]
(is (str/includes? rep "Trend Line"))
(is (str/includes? rep "y =")))))
@@ -126,7 +126,7 @@
(let [series-stats (#'categorical/compute-series-stats ["A" "B" "C"] [100 200 150])
stats {:chart-type :categorical
:series-count 1
- :series {"Test Series" series-stats}}
+ :series [(assoc series-stats :name "Test Series")]}
rep (repr/generate-representation {:stats stats})]
(is (str/includes? rep "Test Series"))
(is (str/includes? rep "Categories"))
@@ -139,7 +139,7 @@
series-stats (#'categorical/compute-series-stats xs ys)
stats {:chart-type :categorical
:series-count 1
- :series {"Many" series-stats}}
+ :series [(assoc series-stats :name "Many")]}
rep (repr/generate-representation {:stats stats})]
(is (str/includes? rep "Top Categories"))
(is (str/includes? rep "Bottom Categories"))
@@ -155,7 +155,7 @@
series-stats (#'categorical/compute-series-stats xs ys)
stats {:chart-type :categorical
:series-count 1
- :series {"Few" series-stats}}
+ :series [(assoc series-stats :name "Few")]}
rep (repr/generate-representation {:stats stats})]
(is (str/includes? rep "Top Categories"))
(is (not (str/includes? rep "Bottom Categories"))))))
@@ -165,7 +165,7 @@
(let [series-stats (#'categorical/compute-series-stats ["A" "B" "C"] [100 200 150])
stats {:chart-type :categorical
:series-count 1
- :series {"Few" series-stats}}
+ :series [(assoc series-stats :name "Few")]}
rep (repr/generate-representation {:stats stats})]
(is (str/includes? rep "limited data points")))))
@@ -183,7 +183,7 @@
(is (some #(str/includes? result %) ["Q1" "Q2" "Q3" "Q4"])))))
(deftest ^:parallel render-trend-no-clear-trend-is-honest-test
- (testing "A :no-clear-trend direction renders honestly — no asserted direction, no confident % headline"
+ (testing "A no-clear-trend direction renders honestly — no asserted direction, no confident % headline"
(let [out (#'repr/render-trend {:direction :no-clear-trend
:overall-change-pct -97.9
:start-value 1727 :end-value 36})]
@@ -199,7 +199,7 @@
series-stats (#'time-series/compute-series-stats values dates {})
stats {:chart-type :time-series
:series-count 1
- :series {"Revenue" series-stats}}
+ :series [(assoc series-stats :name "Revenue")]}
rep (repr/generate-representation {:stats stats})]
(is (str/includes? rep "Revenue"))
(is (str/includes? rep "Trend"))
@@ -207,14 +207,14 @@
(deftest ^:parallel repr-time-series-multi-series-test
(testing "representation with multiple series includes all series names"
- (let [make-stats (fn [values dates]
- (#'time-series/compute-series-stats values dates {}))
+ (let [make-stats (fn [name values dates]
+ (assoc (#'time-series/compute-series-stats values dates {}) :name name))
stats {:chart-type :time-series
:series-count 2
- :series {"Sales" (make-stats [10.0 20.0 30.0 40.0 50.0]
- ["d1" "d2" "d3" "d4" "d5"])
- "Revenue" (make-stats [20.0 40.0 60.0 80.0 100.0]
- ["d1" "d2" "d3" "d4" "d5"])}}
+ :series [(make-stats "Sales" [10.0 20.0 30.0 40.0 50.0]
+ ["d1" "d2" "d3" "d4" "d5"])
+ (make-stats "Revenue" [20.0 40.0 60.0 80.0 100.0]
+ ["d1" "d2" "d3" "d4" "d5"])]}
rep (repr/generate-representation {:stats stats})]
(is (str/includes? rep "Sales"))
(is (str/includes? rep "Revenue")))))
@@ -226,7 +226,7 @@
series-stats (#'time-series/compute-series-stats values dates {:deep? true})
stats {:chart-type :time-series
:series-count 1
- :series {"Metric" series-stats}}
+ :series [(assoc series-stats :name "Metric")]}
rep (repr/generate-representation {:stats stats})]
(is (str/includes? rep "Volatility"))
(is (str/includes? rep "Significant Changes"))
@@ -237,9 +237,10 @@
(deftest ^:parallel generate-representation-dispatches-to-correct-renderer-test
(testing "dispatches to the correct renderer for known chart type"
(let [ts-stats {:chart-type :time-series :series-count 1
- :series {"S" (#'time-series/compute-series-stats
- [1.0 2.0 3.0 4.0 5.0]
- ["d1" "d2" "d3" "d4" "d5"] {})}}]
+ :series [(assoc (#'time-series/compute-series-stats
+ [1.0 2.0 3.0 4.0 5.0]
+ ["d1" "d2" "d3" "d4" "d5"] {})
+ :name "S")]}]
(is (str/includes? (repr/generate-representation {:stats ts-stats}) "Time Series")))))
(deftest ^:parallel generate-representation-unknown-chart-type-fallback-test
diff --git a/test/metabase/interestingness/chart/scatter_test.clj b/test/metabase/interestingness/chart/scatter_test.clj
index 9fcf9d641de..b170bb2c186 100644
--- a/test/metabase/interestingness/chart/scatter_test.clj
+++ b/test/metabase/interestingness/chart/scatter_test.clj
@@ -84,14 +84,16 @@
:display_name "S2"}}]
(is (=? {:chart-type :scatter
:series-count 2
- :series {"S1" {:data-points 3
- :correlation {:coefficient 1.0
- :strength :strong
- :direction :positive}}
- "S2" {:data-points 3
- :correlation {:coefficient -1.0
- :strength :strong
- :direction :negative}}}
+ :series [{:name "S1"
+ :data-points 3
+ :correlation {:coefficient 1.0
+ :strength :strong
+ :direction :positive}}
+ {:name "S2"
+ :data-points 3
+ :correlation {:coefficient -1.0
+ :strength :strong
+ :direction :negative}}]
:correlations (symbol "nil #_\"key is not present.\"")}
(scatter/compute-scatter-stats series-data {}))))))
diff --git a/test/metabase/interestingness/chart/stats_test.clj b/test/metabase/interestingness/chart/stats_test.clj
index abb19f102ba..5e8c3c3fcc8 100644
--- a/test/metabase/interestingness/chart/stats_test.clj
+++ b/test/metabase/interestingness/chart/stats_test.clj
@@ -1,7 +1,11 @@
(ns metabase.interestingness.chart.stats-test
(:require
[clojure.test :refer :all]
- [metabase.interestingness.chart.stats :as stats.core]))
+ [metabase.interestingness.chart.stats :as stats.core]
+ [metabase.interestingness.chart.types :as stats.types]
+ [metabase.models.interface :as mi]
+ [metabase.util.json :as json]
+ [metabase.util.malli.registry :as mr]))
(set! *warn-on-reflection* true)
@@ -27,8 +31,8 @@
;;; ------------------------------------------- Trend-direction schema regression ----------------------------------
(deftest ^:parallel deep-stats-accepts-no-clear-trend-test
- (testing "compute-chart-stats (deep) validates a :no-clear-trend series without throwing"
- ;; reconcile-direction emits :no-clear-trend on a pathologically noisy series (CV ≥ 1);
+ (testing "compute-chart-stats (deep) validates a no-clear-trend series without throwing"
+ ;; reconcile-direction emits "no-clear-trend" on a pathologically noisy series (CV ≥ 1);
;; the ::trend-direction output schema must allow it or deep-stats throws on real data.
(let [config {:display_type "line"
:title "Noisy"
@@ -38,7 +42,7 @@
:x_values ["2020-01-01" "2020-01-02" "2020-01-03" "2020-01-04" "2020-01-05"]
:y_values [10.0 1000.0 5.0 900.0 8.0]}}}
stats (stats.core/compute-chart-stats config {:deep? true})]
- (is (= :no-clear-trend (get-in stats [:series "Noisy" :trend :direction]))))))
+ (is (= :no-clear-trend (get-in stats [:series 0 :trend :direction]))))))
;;; ------------------------------------------- Downsampling Tests -------------------------------------------------
@@ -51,7 +55,7 @@
(deftest ^:parallel data-points-within-limit-are-not-downsampled-test
(testing "Series within the limit are not modified"
(is (=? {:limits (symbol "nil #_\"key is not present.\"")
- :series {"series_0" {:data-points 100}}}
+ :series [{:name "series_0" :data-points 100}]}
(stats.core/compute-chart-stats (make-chart-config 1 100) {:deep? false})))))
(deftest ^:parallel data-points-exceeding-limit-are-downsampled-test
@@ -59,8 +63,8 @@
(let [n (+ @#'stats.core/max-data-points-per-series 5000)
config (make-chart-config 1 n)
stats (stats.core/compute-chart-stats config {:deep? false})
- sampled (get-in stats [:series "series_0" :data-points])]
- (is (=? {:limits {:downsampled-series {"series_0" {:original-count n}}}}
+ sampled (get-in stats [:series 0 :data-points])]
+ (is (=? {:limits {:downsampled-series [{:name "series_0" :original-count n}]}}
stats))
(is (approx=max-data-points-per-series? sampled)
(str "sampled " sampled " should be roughly <= " @#'stats.core/max-data-points-per-series)))))
@@ -70,8 +74,9 @@
(let [n (+ @#'stats.core/max-data-points-per-series 1000)
config (make-chart-config 1 n)
orig-ys (get-in config [:series "series_0" :y_values])]
- (is (=? {:series {"series_0" {:trend {:start-value (double (first orig-ys))
- :end-value (double (last orig-ys))}}}}
+ (is (=? {:series [{:name "series_0"
+ :trend {:start-value (double (first orig-ys))
+ :end-value (double (last orig-ys))}}]}
(stats.core/compute-chart-stats config {:deep? false}))))))
(deftest ^:parallel multiple-series-downsampled-independently-test
@@ -83,11 +88,10 @@
:series {"small" (make-series small-n)
"large" (make-series large-n)}}
stats (stats.core/compute-chart-stats config {:deep? false})]
- (is (=? {:limits {:downsampled-series
- {"small" (symbol "nil #_\"key is not present.\"")
- "large" {:original-count large-n}}}
- :series {"small" {:data-points small-n}
- "large" {:data-points approx=max-data-points-per-series?}}}
+ (is (=? {;; only the oversized series is recorded as downsampled
+ :limits {:downsampled-series [{:name "large" :original-count large-n}]}
+ :series [{:name "small" :data-points small-n}
+ {:name "large" :data-points approx=max-data-points-per-series?}]}
stats)))))
;;; ---------------------------------------- Correlation Cap Tests -------------------------------------------------
@@ -112,3 +116,77 @@
(let [max-k @#'stats.core/max-series-for-correlations
max-pairs (/ (* max-k (dec max-k)) 2)]
(is (<= (count (:correlations stats)) max-pairs))))))
+
+;;; ---------------------------------------- chart-stats-schema codec ----------------------------------------------
+
+(def ^:private codec
+ (mi/transform-json-with-schema "exploration_query_result.chart_stats" stats.core/chart-stats-schema))
+
+(defn- round-trip [stats]
+ ((:out codec) ((:in codec) stats)))
+
+(deftest chart-stats-schema-dispatches-on-chart-type-test
+ (testing "every chart type round-trips through the JSON codec with its keywords intact"
+ (are [stats] (= stats (round-trip stats))
+ {:chart-type :time-series
+ :series-count 1
+ :series [{:name "s"
+ :summary {:min 1.0 :max 9.0 :mean 5.0 :median 5.0 :std-dev 2.5 :range 8.0}
+ :time-range {:start "2020-01-01" :end "2020-02-01" :duration-days 31}
+ :data-points 2
+ :trend {:direction :strongly-increasing :overall-change-pct 12.5}
+ :volatility {:level :moderate :coefficient-of-variation 0.3}
+ :patterns [{:type :spike :description "a spike"}]}]}
+
+ {:chart-type :categorical
+ :series-count 1
+ :series [{:name "s" :summary nil :data-points 1 :category-count 1
+ :top-categories [{:name "ACME" :value 41}]}]}
+
+ {:chart-type :scatter
+ :series-count 1
+ :series [{:name "s" :x-summary nil :y-summary nil :data-points 1}]}
+
+ {:chart-type :histogram
+ :series-count 1
+ :series [{:name "s"
+ :estimated-summary {:weighted-mean 1.0 :weighted-std-dev 2.0 :data-range 3.0}
+ :total-count 5
+ :data-points 1
+ :bin-data [[1.0 2.0]]
+ :distribution {:estimated-percentiles {:p25 1.0 :p50 2.0 :p75 3.0
+ :p90 4.0 :p95 5.0 :p99 6.0}
+ :estimated-quartiles {:q1 1.0 :median 2.0 :q3 3.0 :iqr 2.0}}
+ :structure {:mode-bin [1.0 2.0] :peak-count 1 :concentration-top3 1.0
+ :gap-count 0 :empty-bin-ratio 0.0 :bin-count 1}}]}
+
+ {:chart-type :unknown :series-count 0 :message "not implemented"})))
+
+(deftest chart-stats-schema-is-closed-test
+ (testing "`:unknown` is the only door for a chart type without dedicated analysis, so a typo cannot
+ ride through the `compute-chart-stats` return schema"
+ (is (mr/validate stats.core/chart-stats-schema
+ {:chart-type :unknown :series-count 0 :message "not implemented"}))
+ (is (not (mr/validate stats.core/chart-stats-schema
+ {:chart-type :histogrm :series-count 1 :series [{:garbage true}]})))))
+
+(deftest chart-stats-schema-unrecognized-chart-type-reads-undecoded-test
+ (testing "dropping the default branch tightens validation without hardening the read: a blob with a
+ `:chart-type` this version has no branch for still rides through the decoder untouched,
+ so one such row cannot break a `t2/select`"
+ (is (= {:chart-type "sankey" :series-count 1}
+ ((:out codec) (json/encode {:chart-type "sankey" :series-count 1}))))))
+
+(deftest series-stats-declare-column-labels-test
+ (testing "every chart type's series carries the `:x-name`/`:y-name`
+ [[metabase.interestingness.chart.util/compute-series-with-labels]] attaches, and the
+ schema names them"
+ (let [series (:series (stats.core/compute-chart-stats (make-chart-config 2 30) {:deep? false}))]
+ (is (seq series))
+ (is (every? #(= "Date" (:x-name %)) series))
+ (is (every? #(= "Value" (:y-name %)) series))))
+ (testing "they are optional, though — stats built without that helper are still a valid series,
+ which is why the hand-built fixtures in repr-test render fine"
+ (let [real (first (:series (stats.core/compute-chart-stats (make-chart-config 1 30) {:deep? false})))]
+ (is (mr/validate ::stats.types/time-series-series-stats real))
+ (is (mr/validate ::stats.types/time-series-series-stats (dissoc real :x-name :y-name))))))
diff --git a/test/metabase/interestingness/chart/time_series_test.clj b/test/metabase/interestingness/chart/time_series_test.clj
index 6094ad9de75..1f60cda10b9 100644
--- a/test/metabase/interestingness/chart/time_series_test.clj
+++ b/test/metabase/interestingness/chart/time_series_test.clj
@@ -41,7 +41,7 @@
;;; ---------------------------------------------- compute-trend tests -----------------------------------------------
(deftest ^:parallel compute-trend-strongly-increasing-test
- (testing "strongly increasing series detected as :strongly-increasing"
+ (testing "strongly increasing series detected as strongly-increasing"
;; slope=5, mean=20, pct=100*20/20=100% > 50 → :strongly-increasing
(is (=? {:direction :strongly-increasing
:overall-change-pct pos?
@@ -50,19 +50,19 @@
(#'time-series/compute-trend [10.0 15.0 20.0 25.0 30.0])))))
(deftest ^:parallel compute-trend-strongly-decreasing-test
- (testing "strongly decreasing series detected as :strongly-decreasing"
+ (testing "strongly decreasing series detected as strongly-decreasing"
(is (=? {:direction :strongly-decreasing
:overall-change-pct neg?}
(#'time-series/compute-trend [100.0 80.0 60.0 40.0 20.0])))))
(deftest ^:parallel compute-trend-flat-test
- (testing "roughly flat series detected as :flat"
+ (testing "roughly flat series detected as flat"
(is (=? {:direction :flat}
(#'time-series/compute-trend [50.0 51.0 50.0 50.5 50.0])))))
(deftest ^:parallel compute-trend-increasing-moderate-test
- (testing "moderate increase (10-50% range) detected as :increasing"
- ;; slope=10, mean=120, total_change=40, pct=33.3% → :increasing
+ (testing "moderate increase (10-50% range) detected as increasing"
+ ;; slope=10, mean=120, total_change=40, pct=33.3% → "increasing"
(is (=? {:direction :increasing}
(#'time-series/compute-trend [100.0 110.0 120.0 130.0 140.0])))))
@@ -100,28 +100,28 @@
;;; --------------------------------------------- compute-volatility tests -------------------------------------------
(deftest ^:parallel compute-volatility-low-test
- (testing "tightly clustered values produce :low volatility (cv < 0.1)"
+ (testing "tightly clustered values produce low volatility (cv < 0.1)"
;; mean≈101.6, std≈1.1, cv≈0.011
(is (=? {:level :low
:coefficient-of-variation #(< % 0.1)}
(#'time-series/compute-volatility [100.0 102.0 101.0 103.0 102.0])))))
(deftest ^:parallel compute-volatility-moderate-test
- (testing "moderately variable data produces :moderate volatility (0.1 ≤ cv < 0.3)"
+ (testing "moderately variable data produces moderate volatility (0.1 ≤ cv < 0.3)"
;; mean=100, std≈17.3, cv≈0.173
(is (=? {:level :moderate
:coefficient-of-variation #(and (<= 0.1 %) (< % 0.3))}
(#'time-series/compute-volatility [80.0 100.0 120.0 90.0 110.0])))))
(deftest ^:parallel compute-volatility-high-test
- (testing "fairly variable data produces :high volatility (0.3 ≤ cv < 0.5)"
+ (testing "fairly variable data produces high volatility (0.3 ≤ cv < 0.5)"
;; mean=100, std≈34.6, cv≈0.346
(is (=? {:level :high
:coefficient-of-variation #(and (<= 0.3 %) (< % 0.5))}
(#'time-series/compute-volatility [60.0 100.0 140.0 70.0 130.0])))))
(deftest ^:parallel compute-volatility-extreme-test
- (testing "very variable data produces :extreme volatility (cv ≥ 0.5)"
+ (testing "very variable data produces extreme volatility (cv ≥ 0.5)"
;; mean≈102, std≈75, cv≈0.73
(is (=? {:level :extreme
:coefficient-of-variation #(>= % 0.5)}
@@ -398,10 +398,10 @@
:display_name "S2"}}]
(is (=? {:chart-type :time-series
:series-count 2
- :series {"S1" {:data-points 5
- :trend {:direction :strongly-increasing}}
- "S2" {:data-points 5
- :trend {:direction :strongly-decreasing}}}
+ :series [{:name "S1" :data-points 5
+ :trend {:direction :strongly-increasing}}
+ {:name "S2" :data-points 5
+ :trend {:direction :strongly-decreasing}}]
:correlations (symbol "nil #_\"key is not present.\"")}
(time-series/compute-time-series-stats series-data {}))))))
diff --git a/test/metabase/interestingness/chart/util_test.clj b/test/metabase/interestingness/chart/util_test.clj
index 53cc530b749..6ab27b2362c 100644
--- a/test/metabase/interestingness/chart/util_test.clj
+++ b/test/metabase/interestingness/chart/util_test.clj
@@ -6,8 +6,6 @@
(set! *warn-on-reflection* true)
-;;; ------------------------------------------------ nan->nil -------------------------------------------------------
-
(deftest ^:parallel nan->nil-test
(testing "NaN becomes nil"
(is (nil? (stats.u/nan->nil Double/NaN))))
@@ -15,8 +13,11 @@
(is (= 42.0 (stats.u/nan->nil 42.0)))
(is (= 0.0 (stats.u/nan->nil 0.0)))
(is (= -1.5 (stats.u/nan->nil -1.5))))
- (testing "infinity passes through (not NaN)"
- (is (= Double/POSITIVE_INFINITY (stats.u/nan->nil Double/POSITIVE_INFINITY)))))
+ (testing "the infinities are dropped too, despite the name — as unrepresentable in JSON as NaN.
+ Nothing produces one today (every division that could is zero-guarded), so this pins the
+ guard rather than a live case"
+ (is (nil? (stats.u/nan->nil Double/POSITIVE_INFINITY)))
+ (is (nil? (stats.u/nan->nil Double/NEGATIVE_INFINITY)))))
;;; -------------------------------------------- compute-summary ---------------------------------------------------
@@ -181,10 +182,11 @@
series-data
(fn [xs ys] {:sum (reduce + ys) :count (count xs)}))]
(is (= 1 (count result)))
- (is (=? {"s1" {:sum 60
- :count 3
- :x-name "Date"
- :y-name "Revenue"}}
+ (is (=? [{:name "s1"
+ :sum 60
+ :count 3
+ :x-name "Date"
+ :y-name "Revenue"}]
result)))))
(deftest ^:parallel compute-series-with-labels-nil-metadata-test
@@ -193,8 +195,9 @@
result (stats.u/compute-series-with-labels
series-data
(fn [_ _] {:ok true}))]
- (is (=? {"s1" {:x-name nil?
- :y-name nil?}}
+ (is (=? [{:name "s1"
+ :x-name nil?
+ :y-name nil?}]
result)))))
(deftest ^:parallel compute-series-with-labels-multiple-series-test
@@ -210,8 +213,8 @@
result (stats.u/compute-series-with-labels
series-data
(fn [_ ys] {:val (first ys)}))]
- (is (=? {"a" {:val 10}
- "b" {:val 20}}
+ (is (=? [{:name "a" :val 10}
+ {:name "b" :val 20}]
result)))))
;;; ------------------------------------------- make-chart-result ----------------------------------------------------
@@ -232,27 +235,29 @@
:display_name "Test"})
(def ^:private sample-categorical-series
- {"s1" {:summary sample-summary
- :data-points 5
- :category-count 3
- :top-categories [{:name "A" :value 10.0 :percentage 50.0}]
- :outliers []}})
+ [{:name "s1"
+ :summary sample-summary
+ :data-points 5
+ :category-count 3
+ :top-categories [{:name "A" :value 10.0 :percentage 50.0}]
+ :outliers []}])
(def ^:private sample-histogram-series
- {"s1" {:estimated-summary {:weighted-mean 5.0
- :weighted-std-dev 2.0
- :data-range 9.0}
- :total-count 50
- :data-points 10
- :bin-data [[1.0 5.0] [2.0 10.0]]
- :distribution {:estimated-percentiles {25 2.0 50 5.0 75 8.0 90 9.0 95 9.5 99 9.9}
- :estimated-quartiles {:q1 2.0 :median 5.0 :q3 8.0 :iqr 6.0}}
- :structure {:mode-bin [2.0 10.0]
- :peak-count 1
- :concentration-top3 1.0
- :gap-count 0
- :empty-bin-ratio 0.0
- :bin-count 2}}})
+ [{:name "s1"
+ :estimated-summary {:weighted-mean 5.0
+ :weighted-std-dev 2.0
+ :data-range 9.0}
+ :total-count 50
+ :data-points 10
+ :bin-data [[1.0 5.0] [2.0 10.0]]
+ :distribution {:estimated-percentiles {:p25 2.0 :p50 5.0 :p75 8.0 :p90 9.0 :p95 9.5 :p99 9.9}
+ :estimated-quartiles {:q1 2.0 :median 5.0 :q3 8.0 :iqr 6.0}}
+ :structure {:mode-bin [2.0 10.0]
+ :peak-count 1
+ :concentration-top3 1.0
+ :gap-count 0
+ :empty-bin-ratio 0.0
+ :bin-count 2}}])
(deftest ^:parallel make-chart-result-basic-test
(testing "builds standard chart result with valid schema"
@@ -266,16 +271,18 @@
(testing "includes correlations when provided"
(let [corrs [{:series-a "a" :series-b "b" :coefficient 0.9
:strength :strong :direction :positive :aligned-sample-size 10}]
- series {"a" {:summary sample-summary
- :data-points 2
- :category-count 2
- :top-categories []
- :outliers []}
- "b" {:summary sample-summary
- :data-points 2
- :category-count 2
- :top-categories []
- :outliers []}}
+ series [{:name "a"
+ :summary sample-summary
+ :data-points 2
+ :category-count 2
+ :top-categories []
+ :outliers []}
+ {:name "b"
+ :summary sample-summary
+ :data-points 2
+ :category-count 2
+ :top-categories []
+ :outliers []}]
result (stats.u/make-chart-result :categorical
{"a" sample-series-config "b" sample-series-config}
series corrs)]
diff --git a/test/metabase/models/interface_test.clj b/test/metabase/models/interface_test.clj
index ce6a6a2bb47..baeefb20158 100644
--- a/test/metabase/models/interface_test.clj
+++ b/test/metabase/models/interface_test.clj
@@ -1,6 +1,7 @@
(ns metabase.models.interface-test
{:clj-kondo/config '{:linters {:deprecated-var {:level :off}}}}
(:require
+ [clojure.string :as str]
[clojure.test :refer :all]
[java-time.api :as t]
[metabase.models.interface :as mi]
@@ -9,6 +10,8 @@
[metabase.util.encryption :as encryption]
[metabase.util.encryption-test :as encryption-test]
[metabase.util.json :as json]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.registry-test-macro :as registry-test-macro]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -37,6 +40,53 @@
#"Error decrypting secret\.value: Expected an encrypted value"
((:out (mi/transform-secret-value "secret.value")) (.getBytes "plaintext-sekret")))))))
+(mr/def ::interface-test.thing
+ "A schema with one field of each kind JSON cannot carry on its own."
+ [:map
+ [:kind {:optional true} [:maybe :keyword]]
+ [:tag {:optional true} [:enum :a :b]]
+ [:by-id {:optional true} [:map-of :int :string]]
+ [:note {:optional true} :string]])
+
+(def ^:private a-thing
+ {:kind :time-series, :tag :b, :by-id {10 "x", 20 "y"}, :note "plain"})
+
+(deftest transform-json-with-schema-test
+ (let [{:keys [in out]} (mi/transform-json-with-schema "test_table.thing" ::interface-test.thing)]
+ (testing "the stored value is plain JSON -- every keyword, and every integer map key, is a string"
+ (is (= {"kind" "time-series", "tag" "b", "by-id" {"10" "x", "20" "y"}, "note" "plain"}
+ (json/decode (in a-thing)))))
+ (testing "the schema is what puts the keywords and the integer keys back on the way out"
+ (is (= a-thing (out (in a-thing)))))
+ (testing "nil is SQL NULL rather than the string \"null\", and an empty map stays an empty map"
+ (is (nil? (in nil)))
+ (is (nil? (out nil)))
+ (is (= {} (out (in {})))))
+ (testing "an unreadable blob reads as nil rather than breaking the whole select"
+ (is (nil? (out "{not json ]["))))))
+
+(deftest transform-json-with-schema-caches-its-codec-test
+ (testing "the encoder and decoder are built once per schema, not rebuilt for every row -- these run
+ on every read of every row of a table, and building a transformer for a nested schema is
+ not cheap (see metabase.api.macros, which caches the same way at the HTTP boundary)"
+ (let [{:keys [in out]} (mi/transform-json-with-schema "test_table.thing" ::interface-test.thing)]
+ (out (in a-thing))
+ (is (zero? (registry-test-macro/with-returning-cache-miss-count
+ (dotimes [_ 10]
+ (out (in a-thing)))))))))
+
+(deftest transform-encrypted-json-with-schema-test
+ (encryption-test/with-secret-key "0123456789abcdef"
+ (let [{:keys [in out]} (mi/transform-encrypted-json-with-schema "test_table.thing" ::interface-test.thing)]
+ (testing "the value round-trips through encryption with its keywords intact"
+ (let [stored (in a-thing)]
+ (is (not (str/includes? stored "time-series")))
+ (is (= a-thing (out stored)))))
+ (testing "a decrypt failure names the column, like every other encrypted transform"
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Error decrypting test_table\.thing: Expected an encrypted value"
+ (out "plaintext-sekret")))))))
+
(deftest timestamped-property-test
(testing "Make sure updated_at gets updated for timestamped models"
(mt/with-temp [:model/Table table {:updated_at #t "2023-02-02T01:00:00"}]
diff --git a/test/metabase/permissions/data_access_token_test.clj b/test/metabase/permissions/data_access_token_test.clj
index 704d0bb41c0..45f8b6e7347 100644
--- a/test/metabase/permissions/data_access_token_test.clj
+++ b/test/metabase/permissions/data_access_token_test.clj
@@ -1,6 +1,5 @@
(ns metabase.permissions.data-access-token-test
(:require
- [clojure.edn :as edn]
[clojure.string :as str]
[clojure.test :refer :all]
[metabase.permissions.data-access-token :as data-access-token]
@@ -8,10 +7,10 @@
(def ^:private sandbox-ca {:sandbox {10 "d-ca"}})
(def ^:private sandbox-ny {:sandbox {10 "d-ny"}})
-(def ^:private role-ro {:impersonation {5 {:role "ro"}}})
-(def ^:private role-rw {:impersonation {5 {:role "rw"}}})
-(def ^:private dest-a {:routing {7 {:destination-db-id 100}}})
-(def ^:private dest-b {:routing {7 {:destination-db-id 200}}})
+(def ^:private role-ro {:impersonation {5 "d-ro"}})
+(def ^:private role-rw {:impersonation {5 "d-rw"}})
+(def ^:private dest-a {:routing {7 "d-100"}})
+(def ^:private dest-b {:routing {7 "d-200"}})
(def ^:private unrestricted {})
(deftest data-access-compatible?-sandbox-test
@@ -67,15 +66,13 @@
(deftest data-access-compatible?-multi-table-sandbox-test
(let [compatible? data-access-token/data-access-compatible?
- creator {:sandbox {10 "d-ca"
- 20 "d-west"}}]
+ creator {:sandbox {10 "d-ca" 20 "d-west"}}]
(testing "viewer must match on EVERY touched table"
(is (true? (compatible? creator creator)))
;; matches table 10, unsandboxed on 20 -> blocked (no relaxation for absence)
(is (false? (compatible? creator {:sandbox {10 "d-ca"}})))
;; matches table 10 but a different sandbox on table 20 -> blocked
- (is (false? (compatible? creator {:sandbox {10 "d-ca"
- 20 "d-east"}}))))))
+ (is (false? (compatible? creator {:sandbox {10 "d-ca" 20 "d-east"}}))))))
(deftest data-access-compatible?-oss-test
(testing "two empty (OSS / unrestricted) tokens are always compatible -> no gating"
@@ -103,7 +100,7 @@
:routing {:destination-db-id 100}})
(deftest data-access-token-does-not-retain-raw-lens-values-test
- (testing "the token is persisted as plaintext EDN on stored_result.data_access_token, so no raw
+ (testing "the token is persisted as plaintext JSON on stored_result.data_access_token, so no raw
sandbox attribute value (potentially PII), GTAP card id, or warehouse role may survive in it"
(let [printed (pr-str (token-for ca-lens))]
(doseq [secret ["State" "CA" "CustomerEmail" "[email protected]" "analyst_ro"]]
@@ -153,8 +150,37 @@
(is (false? (data-access-token/data-access-compatible? creator (token-for {}))))
(is (false? (data-access-token/data-access-compatible? (token-for {}) creator)))))))
-(deftest data-access-token-is-edn-round-trippable-test
- (testing "stored_result serializes the token with pr-str and reads it back with a reader-less
- edn/read-string, so every value in it must survive that round trip"
- (let [token (token-for ca-lens)]
- (is (= token (edn/read-string {:readers {} :default (fn [_tag v] v)} (pr-str token)))))))
+(defn- transform-in [token] ((:in data-access-token/data-access-token-transform) token))
+(defn- transform-out [s] ((:out data-access-token/data-access-token-transform) s))
+
+(deftest data-access-token-is-json-round-trippable-test
+ (testing "the token is persisted as JSON and read back identically. JSON has no integer map keys,
+ so the transform decodes against the token schema to restore them — which matters both
+ because the gate is bare `=` and because callers index the dimensions by target id"
+ (doseq [token [(token-for ca-lens) sandbox-ca role-ro dest-a unrestricted
+ {:sandbox {10 "d-ca" 20 "d-west"}}]]
+ (let [stored (transform-in token)]
+ (is (string? stored))
+ (is (= token (transform-out stored)) (pr-str token))))))
+
+(deftest data-access-token-nil-is-sql-null-test
+ (testing "nil is stored as SQL NULL rather than the string \"null\", and an empty (unrestricted)
+ token stays an empty map — nil and {} are not interchangeable to the gate"
+ (is (nil? (transform-in nil)))
+ (is (nil? (transform-out nil)))
+ (is (= {} (transform-out (transform-in {}))))))
+
+(deftest data-access-token-dimensions-are-keyed-by-target-test
+ (testing "each dimension is keyed by its target id, so a caller can ask about one target directly.
+ `metabase.metabot.metadata-perms/row-restricted-by-db` does exactly this to decide whether
+ a table's rows are narrowed, and a shape it cannot index would silently report every table
+ unrestricted"
+ (let [token (do-with-lens {:sandbox [1 "2026-01-01T00:00Z" {"State" "CA"}]}
+ #(data-access-token/data-access-token
+ {:database-id 5 :table-ids #{30 10 20}}))]
+ (is (= #{10 20 30} (set (keys (:sandbox token)))))
+ (is (contains? (:sandbox token) 20)))))
+
+(deftest data-access-token-unparseable-reads-as-nil-test
+ (testing "an unparseable blob fails closed"
+ (is (nil? (transform-out "{not json ][")))))