Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Analyze query executions for metadata (#72726)

John Swanson984e06d3a1bc
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 576c26ac848..3e2bd69ada0 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2417,6 +2417,23 @@
:model/Field
:model/Table}}
+ usage-metadata
+ {:team "UX West"
+ :api #{metabase.usage-metadata.init}
+ :model-imports #{:model/Card
+ :model/Field
+ :model/Query
+ :model/QueryExecution
+ :model/Segment
+ :model/Table}
+ :uses #{app-db
+ lib
+ lib-be
+ models
+ settings
+ task
+ util}}
+
user-key-value
{:team "UX West"
:api #{metabase.user-key-value.api
diff --git a/dev/src/dev/usage_metadata.clj b/dev/src/dev/usage_metadata.clj
new file mode 100644
index 00000000000..991d1791ef6
--- /dev/null
+++ b/dev/src/dev/usage_metadata.clj
@@ -0,0 +1,64 @@
+(ns dev.usage-metadata
+ (:require
+ [java-time.api :as t]
+ [metabase.usage-metadata.batch :as usage-metadata.batch]
+ [metabase.usage-metadata.core :as usage-metadata]
+ [metabase.usage-metadata.models.source-dimension-daily]
+ [metabase.usage-metadata.models.source-dimension-profile-daily]
+ [metabase.usage-metadata.models.source-metric-daily]
+ [metabase.usage-metadata.models.source-segment-daily]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn today-utc
+ "Today's UTC date as a LocalDate."
+ []
+ (t/local-date (t/offset-date-time (t/zone-offset "Z"))))
+
+(defn process-day!
+ "Reprocess `bucket-date` without advancing the usage metadata watermark."
+ [bucket-date]
+ (usage-metadata.batch/reprocess-day! bucket-date))
+
+(defn process-today!
+ "Reprocess today's UTC bucket without advancing the usage metadata watermark."
+ []
+ (process-day! (today-utc)))
+
+(defn rollups-for-day
+ "Fetch usage metadata rollups for `bucket-date`, ordered by descending count."
+ [bucket-date]
+ {:segments (t2/select :model/SourceSegmentDaily
+ :bucket_date bucket-date
+ {:order-by [[:count :desc]]})
+ :metrics (t2/select :model/SourceMetricDaily
+ :bucket_date bucket-date
+ {:order-by [[:count :desc]]})
+ :dimensions (t2/select :model/SourceDimensionDaily
+ :bucket_date bucket-date
+ {:order-by [[:count :desc]]})
+ :profiles (t2/select :model/SourceDimensionProfileDaily
+ :bucket_date bucket-date
+ {:order-by [[:count :desc]]})})
+
+(defn rollup-counts-for-day
+ "Return the number of rollup rows present for `bucket-date`."
+ [bucket-date]
+ {:segments (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)
+ :metrics (t2/count :model/SourceMetricDaily :bucket_date bucket-date)
+ :dimensions (t2/count :model/SourceDimensionDaily :bucket_date bucket-date)
+ :profiles (t2/count :model/SourceDimensionProfileDaily :bucket_date bucket-date)})
+
+(defn implicit-entities-for-table
+ [table-id]
+ (let [opts {:source-type :table :source-id table-id}]
+ {:segments (usage-metadata/implicit-segments opts)
+ :metrics (usage-metadata/implicit-metrics opts)
+ :dimensions (usage-metadata/implicit-dimensions opts)}))
+
+(defn implicit-entities
+ []
+ {:segments (usage-metadata/implicit-segments {})
+ :metrics (usage-metadata/implicit-metrics {})
+ :dimensions (usage-metadata/implicit-dimensions {})})
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index 61bdbd4ceed..fe16e1df3fa 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -135,6 +135,11 @@
"SecurityAdvisory"
"SemanticSearchTokenTracking"
"Session"
+ "SourceDimensionDaily"
+ "SourceDimensionProfileDaily"
+ "SourceMetricDaily"
+ "SourceSegmentCompositeDaily"
+ "SourceSegmentDaily"
"SupportAccessGrantLog"
"TableRemapping"
"TaskHistory"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
index 09ffee3979e..62b117722f3 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
@@ -111,6 +111,11 @@
:model/TableRemapping
:model/Secret
:model/Session
+ :model/SourceDimensionDaily
+ :model/SourceDimensionProfileDaily
+ :model/SourceMetricDaily
+ :model/SourceSegmentCompositeDaily
+ :model/SourceSegmentDaily
:model/SupportAccessGrantLog
:model/TaskHistory
:model/TaskRun
diff --git a/resources/migrations/063/20260610_usage_metadata_composite_rollups.yaml b/resources/migrations/063/20260610_usage_metadata_composite_rollups.yaml
new file mode 100644
index 00000000000..0f00024a539
--- /dev/null
+++ b/resources/migrations/063/20260610_usage_metadata_composite_rollups.yaml
@@ -0,0 +1,80 @@
+databaseChangeLog:
+ - changeSet:
+ id: v63.c3f8a2
+ author: johnswanson
+ comment: Create source_segment_composite_daily for whole-:and filter baskets
+ changes:
+ - createTable:
+ tableName: source_segment_composite_daily
+ remarks: Daily source-scoped composite segment usage rollups (whole :and baskets)
+ columns:
+ - column:
+ name: id
+ type: int
+ autoIncrement: true
+ remarks: Surrogate primary key
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: source_type
+ type: varchar(256)
+ remarks: Owning source kind for direct or projected facts, or null for canonical mixed facts
+ - column:
+ name: source_id
+ type: int
+ remarks: Owning table or card ID for direct or projected facts, or null for canonical mixed facts
+ - column:
+ name: ownership_mode
+ type: varchar(256)
+ remarks: Ownership attribution mode such as direct, projected, or mixed
+ constraints:
+ nullable: false
+ - column:
+ name: clause
+ type: ${text.type}
+ remarks: Canonical serialized representation of the whole :and filter clause
+ constraints:
+ nullable: false
+ - column:
+ name: atom_fingerprints
+ type: ${text.type}
+ remarks: Sorted JSON array of canonical serialized atom clauses contained in the basket
+ constraints:
+ nullable: false
+ - column:
+ name: atom_count
+ type: int
+ remarks: Number of atomic predicates in the basket
+ constraints:
+ nullable: false
+ - column:
+ name: bucket_date
+ type: date
+ remarks: UTC day covered by the rollup row
+ constraints:
+ nullable: false
+ - column:
+ name: count
+ type: int
+ remarks: Number of observations for this composite fact on the bucket day
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v63.e71b4d
+ author: johnswanson
+ comment: Add source_segment_composite_daily bucket/source index
+ changes:
+ - createIndex:
+ indexName: idx_source_segment_composite_daily_bucket_source
+ tableName: source_segment_composite_daily
+ columns:
+ - column:
+ name: bucket_date
+ - column:
+ name: source_type
+ - column:
+ name: source_id
+ - column:
+ name: ownership_mode
diff --git a/resources/migrations/063/20260610_usage_metadata_rollups.yaml b/resources/migrations/063/20260610_usage_metadata_rollups.yaml
new file mode 100644
index 00000000000..3a6f5e42325
--- /dev/null
+++ b/resources/migrations/063/20260610_usage_metadata_rollups.yaml
@@ -0,0 +1,364 @@
+databaseChangeLog:
+ - changeSet:
+ id: v63.Oong1I
+ author: johnswanson
+ comment: Create source_segment_daily for usage metadata rollups
+ changes:
+ - createTable:
+ tableName: source_segment_daily
+ remarks: Daily source-scoped segment usage rollups
+ columns:
+ - column:
+ name: id
+ type: int
+ autoIncrement: true
+ remarks: Surrogate primary key
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: source_type
+ type: varchar(254)
+ remarks: Owning source kind for direct or projected facts, or null for canonical mixed facts
+ - column:
+ name: source_id
+ type: int
+ remarks: Owning table or card ID for direct or projected facts, or null for canonical mixed facts
+ - column:
+ name: ownership_mode
+ type: varchar(254)
+ remarks: Ownership attribution mode such as direct, projected, or mixed
+ constraints:
+ nullable: false
+ - column:
+ name: field_id
+ type: int
+ remarks: Field referenced by the segment fact when applicable
+ - column:
+ name: predicate
+ type: ${text.type}
+ remarks: Canonical serialized representation of the atomic segment predicate
+ constraints:
+ nullable: false
+ - column:
+ name: bucket_date
+ type: date
+ remarks: UTC day covered by the rollup row
+ constraints:
+ nullable: false
+ - column:
+ name: count
+ type: int
+ remarks: Number of observations for this fact on the bucket day
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v63.Xoo5ue
+ author: johnswanson
+ comment: Create source_metric_daily for usage metadata rollups
+ changes:
+ - createTable:
+ tableName: source_metric_daily
+ remarks: Daily source-scoped metric usage rollups
+ columns:
+ - column:
+ name: id
+ type: int
+ autoIncrement: true
+ remarks: Surrogate primary key
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: source_type
+ type: varchar(254)
+ remarks: Owning source kind for direct or projected facts, or null for canonical mixed facts
+ - column:
+ name: source_id
+ type: int
+ remarks: Owning table or card ID for direct or projected facts, or null for canonical mixed facts
+ - column:
+ name: ownership_mode
+ type: varchar(254)
+ remarks: Ownership attribution mode such as direct, projected, or mixed
+ constraints:
+ nullable: false
+ - column:
+ name: agg_type
+ type: varchar(254)
+ remarks: Aggregation operator for the metric fact
+ constraints:
+ nullable: false
+ - column:
+ name: agg_field_id
+ type: int
+ remarks: Aggregated field referenced by the metric fact when applicable
+ - column:
+ name: temporal_field_id
+ type: int
+ remarks: Breakout field supplying temporal context for the metric fact when applicable
+ - column:
+ name: temporal_unit
+ type: varchar(254)
+ remarks: Temporal bucketing unit associated with the metric fact when applicable
+ - column:
+ name: bucket_date
+ type: date
+ remarks: UTC day covered by the rollup row
+ constraints:
+ nullable: false
+ - column:
+ name: count
+ type: int
+ remarks: Number of observations for this fact on the bucket day
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v63.Tahg3l
+ author: johnswanson
+ comment: Create source_dimension_daily for usage metadata rollups
+ changes:
+ - createTable:
+ tableName: source_dimension_daily
+ remarks: Daily source-scoped dimension usage rollups
+ columns:
+ - column:
+ name: id
+ type: int
+ autoIncrement: true
+ remarks: Surrogate primary key
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: source_type
+ type: varchar(254)
+ remarks: Owning source kind for the dimension fact
+ constraints:
+ nullable: false
+ - column:
+ name: source_id
+ type: int
+ remarks: Owning table or card ID for the dimension fact
+ constraints:
+ nullable: false
+ - column:
+ name: ownership_mode
+ type: varchar(254)
+ remarks: Ownership attribution mode for the dimension fact
+ constraints:
+ nullable: false
+ - column:
+ name: field_id
+ type: int
+ remarks: Breakout field referenced by the dimension fact
+ constraints:
+ nullable: false
+ - column:
+ name: temporal_unit
+ type: varchar(254)
+ remarks: Temporal bucketing unit for temporal breakouts when applicable
+ - column:
+ name: binning
+ type: ${text.type}
+ remarks: Canonical serialized binning configuration when applicable
+ - column:
+ name: bucket_date
+ type: date
+ remarks: UTC day covered by the rollup row
+ constraints:
+ nullable: false
+ - column:
+ name: count
+ type: int
+ remarks: Number of observations for this fact on the bucket day
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v63.ojei4O
+ author: johnswanson
+ comment: Create source_dimension_profile_daily for usage metadata profile observations
+ changes:
+ - createTable:
+ tableName: source_dimension_profile_daily
+ remarks: Daily source-scoped dimension profile observations
+ columns:
+ - column:
+ name: id
+ type: int
+ autoIncrement: true
+ remarks: Surrogate primary key
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: source_type
+ type: varchar(254)
+ remarks: Owning source kind for the profiled dimension
+ constraints:
+ nullable: false
+ - column:
+ name: source_id
+ type: int
+ remarks: Owning table or card ID for the profiled dimension
+ constraints:
+ nullable: false
+ - column:
+ name: field_id
+ type: int
+ remarks: Field being profiled
+ constraints:
+ nullable: false
+ - column:
+ name: source_basis
+ type: varchar(254)
+ remarks: Provenance of the profile signal such as fingerprint
+ constraints:
+ nullable: false
+ - column:
+ name: observation_type
+ type: varchar(254)
+ remarks: Kind of profile observation such as single-value or all-null
+ constraints:
+ nullable: false
+ - column:
+ name: observation_value
+ type: ${text.type}
+ remarks: Optional serialized value associated with the profile observation
+ - column:
+ name: bucket_date
+ type: date
+ remarks: UTC day when the profile observation was recorded
+ constraints:
+ nullable: false
+ - column:
+ name: count
+ type: int
+ remarks: Number of times the observation was recorded on the bucket day
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v63.Ohk8ph
+ author: johnswanson
+ comment: Add source_segment_daily bucket/source index
+ changes:
+ - createIndex:
+ indexName: idx_source_segment_daily_bucket_source
+ tableName: source_segment_daily
+ columns:
+ - column:
+ name: bucket_date
+ - column:
+ name: source_type
+ - column:
+ name: source_id
+ - column:
+ name: ownership_mode
+
+ - changeSet:
+ id: v63.Uike3b
+ author: johnswanson
+ comment: Add source_segment_daily field index
+ changes:
+ - createIndex:
+ indexName: idx_source_segment_daily_field
+ tableName: source_segment_daily
+ columns:
+ - column:
+ name: field_id
+
+ - changeSet:
+ id: v63.Chuun4
+ author: johnswanson
+ comment: Add source_metric_daily bucket/source index
+ changes:
+ - createIndex:
+ indexName: idx_source_metric_daily_bucket_source
+ tableName: source_metric_daily
+ columns:
+ - column:
+ name: bucket_date
+ - column:
+ name: source_type
+ - column:
+ name: source_id
+ - column:
+ name: ownership_mode
+
+ - changeSet:
+ id: v63.eiQu2a
+ author: johnswanson
+ comment: Add source_metric_daily aggregation index
+ changes:
+ - createIndex:
+ indexName: idx_source_metric_daily_agg
+ tableName: source_metric_daily
+ columns:
+ - column:
+ name: agg_type
+ - column:
+ name: agg_field_id
+
+ - changeSet:
+ id: v63.eeQu6e
+ author: johnswanson
+ comment: Add source_dimension_daily bucket/source index
+ changes:
+ - createIndex:
+ indexName: idx_source_dimension_daily_bucket_source
+ tableName: source_dimension_daily
+ columns:
+ - column:
+ name: bucket_date
+ - column:
+ name: source_type
+ - column:
+ name: source_id
+ - column:
+ name: ownership_mode
+
+ - changeSet:
+ id: v63.mohCo0
+ author: johnswanson
+ comment: Add source_dimension_daily field index
+ changes:
+ - createIndex:
+ indexName: idx_source_dimension_daily_field
+ tableName: source_dimension_daily
+ columns:
+ - column:
+ name: field_id
+
+ - changeSet:
+ id: v63.Ye2eib
+ author: johnswanson
+ comment: Add source_dimension_profile_daily bucket/source index
+ changes:
+ - createIndex:
+ indexName: idx_source_dimension_profile_daily_bucket_source
+ tableName: source_dimension_profile_daily
+ columns:
+ - column:
+ name: bucket_date
+ - column:
+ name: source_type
+ - column:
+ name: source_id
+
+ - changeSet:
+ id: v63.ouZ0sh
+ author: johnswanson
+ comment: Add source_dimension_profile_daily field/observation index
+ changes:
+ - createIndex:
+ indexName: idx_source_dimension_profile_daily_field
+ tableName: source_dimension_profile_daily
+ columns:
+ - column:
+ name: field_id
+ - column:
+ name: observation_type
diff --git a/src/metabase/core/init.clj b/src/metabase/core/init.clj
index 467ea412dbb..5d7a3752330 100644
--- a/src/metabase/core/init.clj
+++ b/src/metabase/core/init.clj
@@ -77,6 +77,7 @@
[metabase.transforms.init]
[metabase.types.init]
[metabase.upload.init]
+ [metabase.usage-metadata.init]
[metabase.user-key-value.init]
[metabase.users.init]
[metabase.version.init]
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index d5bf5c012e7..32bc406bc80 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -100,6 +100,7 @@
[metabase.lib.query :as lib.query]
[metabase.lib.query.test-spec :as lib.query.test-spec]
[metabase.lib.ref :as lib.ref]
+ [metabase.lib.referenced-columns :as lib.referenced-columns]
[metabase.lib.remove-replace :as lib.remove-replace]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.aggregation :as lib.schema.aggregation]
@@ -210,6 +211,7 @@
lib.query/keep-me
lib.query.test-spec/keep-me
lib.ref/keep-me
+ lib.referenced-columns/keep-me
lib.remove-replace/keep-me
lib.schema/keep-me
metabase.lib.schema.util/keep-me
@@ -671,6 +673,17 @@
stage-number :- :int]
(lib.filter/filters a-query stage-number)))
+(mu/defn atomic-filters :- [:maybe [:ref ::lib.schema/filters]]
+ "Like [[filters]], but with any top-level `:and` clauses recursively flattened so the result is a
+ list of atomic (non-`:and`) boolean filter clauses. The conjunction of the returned list is
+ logically equivalent to the conjunction of [[filters]].
+
+ **Code Health:** Healthy. This is a core API."
+ ([a-query] (atomic-filters a-query -1))
+ ([a-query :- ::lib.schema/query
+ stage-number :- :int]
+ (lib.filter/atomic-filters a-query stage-number)))
+
(mu/defn filterable-columns :- [:maybe [:sequential ::lib.schema.metadata/column]]
"Returns column metadata for all the columns that could be used for filtering on the target stage of `a-query`.
@@ -1565,6 +1578,8 @@
field-ref-id
field-ref-name
ref]
+ [lib.referenced-columns
+ referenced-columns]
[lib.remove-replace
remove-clause
remove-join
diff --git a/src/metabase/lib/filter.cljc b/src/metabase/lib/filter.cljc
index f5457516069..df190181396 100644
--- a/src/metabase/lib/filter.cljc
+++ b/src/metabase/lib/filter.cljc
@@ -477,6 +477,21 @@
stage-number :- [:maybe :int]]
(perf/not-empty (:filters (lib.util/query-stage query (clojure.core/or stage-number -1))))))
+(defn- flatten-and-clause
+ [a-filter]
+ (if (lib.util/clause-of-type? a-filter :and)
+ (into [] (mapcat flatten-and-clause) (drop 2 a-filter))
+ [a-filter]))
+
+(mu/defn atomic-filters :- [:maybe [:ref ::lib.schema/filters]]
+ "Like [[filters]], but with any top-level `:and` clauses recursively flattened so the result is a
+ list of atomic (non-`:and`) boolean filter clauses. The conjunction of the returned list is
+ logically equivalent to the conjunction of [[filters]]."
+ ([query :- ::lib.schema/query] (atomic-filters query nil))
+ ([query :- ::lib.schema/query
+ stage-number :- [:maybe :int]]
+ (perf/not-empty (into [] (mapcat flatten-and-clause) (filters query stage-number)))))
+
(defn- leading-ref
"Returns the first argument of `a-filter` if it is a reference clause, nil otherwise."
[a-filter]
diff --git a/src/metabase/lib/referenced_columns.cljc b/src/metabase/lib/referenced_columns.cljc
new file mode 100644
index 00000000000..5b7d15986a0
--- /dev/null
+++ b/src/metabase/lib/referenced_columns.cljc
@@ -0,0 +1,46 @@
+(ns metabase.lib.referenced-columns
+ "The natural home for `referenced-columns` would be next to `returned-columns` and friends in
+ `metabase.lib.metadata.calculation`, but the implementation is built on
+ [[metabase.lib.fe-util/expression-parts]], and requiring [[metabase.lib.fe-util]] from there
+ would create a dependency cycle. So it lives in its own namespace downstream of `fe-util`
+ instead."
+ (:require
+ [metabase.lib.dispatch :as lib.dispatch]
+ [metabase.lib.fe-util :as lib.fe-util]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
+ [metabase.lib.util :as lib.util]
+ [metabase.util.malli :as mu]))
+
+(defn- column-metadata?
+ [x]
+ (and (map? x) (= :metadata/column (:lib/type x))))
+
+(defn- expression-parts-map?
+ [x]
+ (and (map? x) (= :mbql/expression-parts (:lib/type x))))
+
+(defn- referenced-columns* [parts]
+ (cond
+ (column-metadata? parts) [parts]
+ (expression-parts-map? parts) (into [] (mapcat referenced-columns*) (:args parts))
+ :else []))
+
+(mu/defn referenced-columns :- [:sequential ::lib.schema.metadata/column]
+ "Returns the column-metadata leaves referenced by `x`, where `x` is a clause (filter, aggregation,
+ expression) or a column-metadata map, resolved relative to the target stage of `query`.
+
+ Columns appear in traversal order; duplicates are NOT removed — callers may `distinct` if
+ dedup semantics are needed.
+
+ Throws if `x` is neither an MBQL clause nor a `:metadata/column` map. Schema validation is
+ off in production, so the runtime check is the gate."
+ ([query x] (referenced-columns query -1 x))
+ ([query :- ::lib.schema/query
+ stage-number :- :int
+ x]
+ (when-not (or (column-metadata? x) (lib.util/clause? x))
+ (throw (ex-info "referenced-columns: expected an MBQL clause or :metadata/column"
+ {:dispatch-value (lib.dispatch/dispatch-value x)
+ :input x})))
+ (referenced-columns* (lib.fe-util/expression-parts query stage-number x))))
diff --git a/src/metabase/lib/schema/info.cljc b/src/metabase/lib/schema/info.cljc
index b7341eb83f8..f3bf289166e 100644
--- a/src/metabase/lib/schema/info.cljc
+++ b/src/metabase/lib/schema/info.cljc
@@ -19,6 +19,7 @@
:cache-refresh
:collection
:map-tiles
+ :metric
:pulse
:dashboard-subscription
:dashboard
diff --git a/src/metabase/lib/schema/mbql_clause.cljc b/src/metabase/lib/schema/mbql_clause.cljc
index 92f1661e287..f3b5f2055a0 100644
--- a/src/metabase/lib/schema/mbql_clause.cljc
+++ b/src/metabase/lib/schema/mbql_clause.cljc
@@ -101,9 +101,12 @@
(defn- normalize-clause [x]
(when (sequential? x)
- (if ((some-fn map? nil?) (second x))
- x
- (into [(first x) {:lib/uuid (str (random-uuid))}] (rest x)))))
+ ;; Coerce non-vector sequentials (e.g. LazySeqs from `json/decode+kw`) to vectors so downstream
+ ;; `:tuple` schemas match.
+ (let [x (if (vector? x) x (vec x))]
+ (if ((some-fn map? nil?) (second x))
+ x
+ (into [(first x) {:lib/uuid (str (random-uuid))}] (rest x))))))
;;; TODO: Support options more nicely - these don't allow for overriding the options, but we have a few cases where that
;;; is necessary. See for example the inclusion of `string-filter-options` in [[metabase.lib.filter]].
diff --git a/src/metabase/metrics/api.clj b/src/metabase/metrics/api.clj
index 8e8cfd22c58..bb4f78ea318 100644
--- a/src/metabase/metrics/api.clj
+++ b/src/metabase/metrics/api.clj
@@ -187,6 +187,15 @@
:projections (or projections [])
:metadata-provider provider}))
+(defn- with-metric-exec-info
+ "Attach `:executed-by`/`:context` to a metric-plan MBQL query so that
+ [[metabase.query-processor.middleware.process-userland-query]] will write a
+ `QueryExecution` row. Without this, the middleware logs
+ \"Cannot save QueryExecution, missing :context\" and skips the insert."
+ [query]
+ (update query :info merge {:executed-by api/*current-user-id*
+ :context :metric}))
+
(defn- execute-leaf-queries
"Execute all leaf queries in parallel, collecting results eagerly.
Must be called OUTSIDE streaming context to avoid JSON writer conflicts.
@@ -194,7 +203,7 @@
[leaves]
(let [uuid->future (into {}
(map (fn [[uuid leaf-plan]]
- [uuid (future (qp/process-query (qp/userland-query (:leaf/mbql leaf-plan))))]))
+ [uuid (future (qp/process-query (qp/userland-query (with-metric-exec-info (:leaf/mbql leaf-plan)))))]))
leaves)]
(into {}
(map (fn [[uuid f]] [uuid @f]))
@@ -226,7 +235,7 @@
plan (lib-metric/->query-plan definition {:limit 10000})]
(if (= :leaf (:plan/type plan))
(qp.streaming/streaming-response [rff :api]
- (qp/process-query (qp/userland-query (:plan/mbql plan)) rff))
+ (qp/process-query (qp/userland-query (with-metric-exec-info (:plan/mbql plan))) rff))
;; Arithmetic: execute leaf queries BEFORE streaming to avoid JSON writer conflicts
(let [uuid->result (execute-leaf-queries (:plan/leaves plan))]
(qp.streaming/streaming-response [rff :api]
@@ -250,7 +259,7 @@
{:keys [definition]} :- ::DatasetRequest]
(let [definition (from-api-definition (lib-metric/metadata-provider) definition)
plan (lib-metric/->query-plan definition {:limit 100 :values-only true})
- result (qp/process-query (qp/userland-query (:plan/mbql plan)))
+ result (qp/process-query (qp/userland-query (with-metric-exec-info (:plan/mbql plan))))
col (first (get-in result [:data :cols]))
values (mapv first (get-in result [:data :rows]))]
{:values values
diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.clj
index d905f02c4fd..00e4ed4ff63 100644
--- a/src/metabase/models/resolution.clj
+++ b/src/metabase/models/resolution.clj
@@ -130,6 +130,11 @@
:model/TransformTag metabase.transforms.models.transform-tag
:model/TransformTransformTag metabase.transforms.models.transform-transform-tag
:model/Undo metabase-enterprise.action-v2.models.undo
+ :model/SourceDimensionDaily metabase.usage-metadata.models.source-dimension-daily
+ :model/SourceDimensionProfileDaily metabase.usage-metadata.models.source-dimension-profile-daily
+ :model/SourceMetricDaily metabase.usage-metadata.models.source-metric-daily
+ :model/SourceSegmentCompositeDaily metabase.usage-metadata.models.source-segment-composite-daily
+ :model/SourceSegmentDaily metabase.usage-metadata.models.source-segment-daily
:model/User metabase.users.models.user
:model/UserKeyValue metabase.user-key-value.models.user-key-value
:model/UserParameterValue metabase.users.models.user-parameter-value
diff --git a/src/metabase/usage_metadata/batch.clj b/src/metabase/usage_metadata/batch.clj
new file mode 100644
index 00000000000..8863db4f088
--- /dev/null
+++ b/src/metabase/usage_metadata/batch.clj
@@ -0,0 +1,391 @@
+(ns metabase.usage-metadata.batch
+ (:require
+ [java-time.api :as t]
+ [metabase.app-db.core :as mdb]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.usage-metadata.extract :as usage-metadata.extract]
+ [metabase.usage-metadata.models.source-dimension-daily]
+ [metabase.usage-metadata.models.source-dimension-profile-daily]
+ [metabase.usage-metadata.models.source-metric-daily]
+ [metabase.usage-metadata.models.source-segment-composite-daily]
+ [metabase.usage-metadata.models.source-segment-daily]
+ [metabase.usage-metadata.settings :as usage-metadata.settings]
+ [metabase.usage-metadata.store :as usage-metadata.store]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2]
+ [toucan2.realize :as t2.realize])
+ (:import
+ (java.nio ByteBuffer)
+ (org.postgresql.util PGobject)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private utc-zone-offset
+ (t/zone-offset "Z"))
+
+(def ^:private rollup-models
+ [:model/SourceSegmentDaily
+ :model/SourceSegmentCompositeDaily
+ :model/SourceMetricDaily
+ :model/SourceDimensionDaily
+ :model/SourceDimensionProfileDaily])
+
+(def ^:private low-cardinality-threshold
+ 30)
+
+(def ^:private hash-chunk-size
+ "Maximum number of query hashes to put into a single `IN (?, ?, ...)` clause.
+ Keeps us safely below the Postgres ~65535 bind-parameter cap when a busy
+ instance accumulates a huge number of unique query hashes per day."
+ 10000)
+
+(defn- today-utc []
+ (t/local-date (t/offset-date-time utc-zone-offset)))
+
+(defn- yesterday-utc []
+ (t/minus (today-utc) (t/days 1)))
+
+(defn- parse-watermark [watermark]
+ (when (seq watermark)
+ (t/local-date watermark)))
+
+(defn- hash-key [hash-bytes]
+ (when hash-bytes
+ (ByteBuffer/wrap ^bytes hash-bytes)))
+
+(defn target-days
+ "Return the closed UTC days the scheduled batch should process."
+ ([] (target-days {:last-completed-day (parse-watermark (usage-metadata.settings/usage-metadata-last-completed-day))
+ :retention-days (usage-metadata.settings/usage-metadata-retention-days)
+ :yesterday (yesterday-utc)}))
+ ([{:keys [last-completed-day retention-days yesterday]}]
+ (let [retention-days (max 1 (or retention-days 1))
+ yesterday (or yesterday (yesterday-utc))
+ retention-start (t/minus yesterday (t/days (dec retention-days)))
+ next-day (some-> last-completed-day (t/plus (t/days 1)))
+ start-day (if next-day
+ (if (neg? (compare retention-start next-day))
+ next-day
+ retention-start)
+ retention-start)]
+ (loop [day start-day
+ days []]
+ (if (pos? (compare day yesterday))
+ days
+ (recur (t/plus day (t/days 1))
+ (conj days day)))))))
+
+(defn- utc-day-start [bucket-date]
+ (t/offset-date-time bucket-date (t/local-time 0) utc-zone-offset))
+
+(defn- utc-day-end [bucket-date]
+ (-> bucket-date
+ utc-day-start
+ (t/plus (t/days 1))))
+
+(defn- execution-hash-counts-for-day
+ "Return `[{:hash byte[], :n execution-count} ...]` for a UTC day.
+
+ Identical hashes produce identical facts, so collapse at the DB and iterate
+ unique hashes. Per-execution totals come from multiplying by `:n`."
+ [bucket-date]
+ (t2/select [:model/QueryExecution :hash [:%count.* :n]]
+ {:where [:and
+ [:>= :started_at (utc-day-start bucket-date)]
+ [:< :started_at (utc-day-end bucket-date)]]
+ :group-by [:hash]}))
+
+(defn- add-skip [stats reason n]
+ (update-in stats [:skipped-rows reason] (fnil + 0) n))
+
+(defn- query->facts [stored-query]
+ (try
+ (let [normalized-query (lib-be/normalize-query stored-query)]
+ (cond
+ (empty? normalized-query)
+ {:status :skip, :reason :normalize-error}
+
+ (nil? (usage-metadata.extract/query-source-table-or-card normalized-query))
+ {:status :skip, :reason :unsupported-query}
+
+ :else
+ (try
+ {:status :ok
+ :facts (usage-metadata.extract/extract-usage-facts normalized-query)}
+ (catch Throwable _
+ {:status :skip, :reason :extract-error}))))
+ (catch Throwable _
+ {:status :skip, :reason :normalize-error})))
+
+(defn- aggregate-row [rows row n]
+ (update rows row (fnil + 0) n))
+
+(defn- segment-row [bucket-date {:keys [source-type source-id ownership-mode field-id predicate]}]
+ {:source_type source-type
+ :source_id source-id
+ :ownership_mode ownership-mode
+ :field_id field-id
+ :predicate predicate
+ :bucket_date bucket-date})
+
+(defn- composite-row [bucket-date {:keys [source-type source-id ownership-mode clause atom-fingerprints atom-count]}]
+ {:source_type source-type
+ :source_id source-id
+ :ownership_mode ownership-mode
+ :clause clause
+ :atom_fingerprints (json/encode atom-fingerprints)
+ :atom_count atom-count
+ :bucket_date bucket-date})
+
+(defn- metric-row [bucket-date {:keys [source-type source-id ownership-mode agg agg-field-id temporal-field-id temporal-unit]}]
+ {:source_type source-type
+ :source_id source-id
+ :ownership_mode ownership-mode
+ :agg_type agg
+ :agg_field_id agg-field-id
+ :temporal_field_id temporal-field-id
+ :temporal_unit temporal-unit
+ :bucket_date bucket-date})
+
+(defn- dimension-row [bucket-date {:keys [source-type source-id ownership-mode field-id temporal-unit binning]}]
+ {:source_type source-type
+ :source_id source-id
+ :ownership_mode ownership-mode
+ :field_id field-id
+ :temporal_unit temporal-unit
+ :binning binning
+ :bucket_date bucket-date})
+
+(defn- counts->rows [counts]
+ (mapv (fn [[row count]]
+ (assoc row :count count))
+ counts))
+
+(defn- accumulate-query-result [stats bucket-date n result]
+ (if (= :ok (:status result))
+ (let [{:keys [segments composites metrics dimensions]} (:facts result)
+ add-rows (fn [acc rows row-fn]
+ (reduce (fn [m fact] (aggregate-row m (row-fn bucket-date fact) n))
+ acc
+ rows))]
+ (-> stats
+ (update :segment-tuples + (* n (count segments)))
+ (update :composite-tuples + (* n (count composites)))
+ (update :metric-tuples + (* n (count metrics)))
+ (update :dimension-tuples + (* n (count dimensions)))
+ (update :segments add-rows segments segment-row)
+ (update :composites add-rows composites composite-row)
+ (update :metrics add-rows metrics metric-row)
+ (update :dimensions add-rows dimensions dimension-row)))
+ (add-skip stats (:reason result) n)))
+
+(defn- fingerprint-observations
+ [fingerprint]
+ (let [distinct-count (get-in fingerprint [:global :distinct-count])
+ nil-pct (get-in fingerprint [:global :nil%])]
+ (cond-> []
+ (= 1 distinct-count)
+ (conj {:observation_type :single-value
+ :observation_value nil})
+
+ (= 1.0 nil-pct)
+ (conj {:observation_type :all-null
+ :observation_value nil})
+
+ (and (integer? distinct-count)
+ (> distinct-count 1)
+ (<= distinct-count low-cardinality-threshold)
+ (or (nil? nil-pct) (< nil-pct 1.0)))
+ (conj {:observation_type :low-cardinality
+ :observation_value (str distinct-count)}))))
+
+(defn- decode-fingerprint
+ [value]
+ (cond
+ (map? value)
+ value
+
+ (instance? PGobject value)
+ (some-> ^PGobject value .getValue json/decode+kw)
+
+ (string? value)
+ (json/decode+kw value)
+
+ :else
+ nil))
+
+(defn- field-fingerprint
+ [field-id]
+ (some-> (t2/select-one-fn :fingerprint :metabase_field :id field-id)
+ decode-fingerprint))
+
+(defn- profile-rows-for-dimensions
+ [bucket-date dimension-rows]
+ (into []
+ (mapcat (fn [{:keys [source_type source_id ownership_mode field_id count]}]
+ (when (and (= source_type :table)
+ (= ownership_mode :direct)
+ (pos-int? field_id))
+ (when-let [fingerprint (field-fingerprint field_id)]
+ (for [{:keys [observation_type observation_value]} (fingerprint-observations fingerprint)]
+ {:source_type source_type
+ :source_id source_id
+ :field_id field_id
+ :source_basis :fingerprint
+ :observation_type observation_type
+ :observation_value observation_value
+ :bucket_date bucket-date
+ :count count})))))
+ dimension-rows))
+
+(defn- process-query-row
+ [bucket-date hash->count stats {:keys [query_hash query]}]
+ (let [hash-bb (hash-key query_hash)
+ n (get hash->count hash-bb)
+ stats (update stats :seen-hashes conj hash-bb)]
+ (if n
+ (-> stats
+ (update :joined-rows + n)
+ (accumulate-query-result bucket-date n (query->facts query)))
+ stats)))
+
+(defn process-day!
+ "Process one UTC bucket day.
+
+ By default this advances the scheduled watermark after a successful replace-day write.
+ Pass `{:advance-watermark? false}` for manual reprocessing of any day without changing
+ scheduled batch state."
+ ([bucket-date]
+ (process-day! bucket-date {:advance-watermark? true}))
+ ([bucket-date {:keys [advance-watermark?]
+ :or {advance-watermark? true}}]
+ (let [hash-counts (execution-hash-counts-for-day bucket-date)
+ raw-hashes (mapv :hash hash-counts)
+ hash->count (into {} (map (fn [{:keys [hash n]}] [(hash-key hash) n])) hash-counts)
+ initial-stats {:bucket-date bucket-date
+ :query-execution-rows (reduce + 0 (vals hash->count))
+ :joined-rows 0
+ :segment-tuples 0
+ :composite-tuples 0
+ :metric-tuples 0
+ :dimension-tuples 0
+ :profile-observations 0
+ :skipped-rows {}
+ :segments {}
+ :composites {}
+ :metrics {}
+ :dimensions {}
+ :seen-hashes #{}}
+ after-stream (reduce
+ (fn [stats hash-chunk]
+ (transduce
+ (map t2.realize/realize)
+ (completing (partial process-query-row bucket-date hash->count))
+ stats
+ (mdb/streaming-reducible
+ (fn [conn]
+ (t2/reducible-select :conn conn [:model/Query :query_hash :query]
+ :query_hash [:in hash-chunk])))))
+ initial-stats
+ (partition-all hash-chunk-size raw-hashes))
+ seen-hashes (:seen-hashes after-stream)
+ day-stats (reduce-kv
+ (fn [stats hash-bb n]
+ (if (contains? seen-hashes hash-bb)
+ stats
+ (add-skip stats :missing-query n)))
+ (dissoc after-stream :seen-hashes)
+ hash->count)
+ dimension-rows (counts->rows (:dimensions day-stats))
+ profile-rows (profile-rows-for-dimensions bucket-date dimension-rows)
+ payload {:segments (counts->rows (:segments day-stats))
+ :composites (counts->rows (:composites day-stats))
+ :metrics (counts->rows (:metrics day-stats))
+ :dimensions dimension-rows
+ :profiles profile-rows}]
+ (usage-metadata.store/replace-day! bucket-date payload)
+ (when advance-watermark?
+ (usage-metadata.settings/usage-metadata-last-completed-day! (str bucket-date)))
+ (assoc day-stats
+ :watermark-advanced? advance-watermark?
+ :segment-rollup-rows (count (:segments payload))
+ :composite-rollup-rows (count (:composites payload))
+ :metric-rollup-rows (count (:metrics payload))
+ :dimension-rollup-rows (count (:dimensions payload))
+ :profile-observations (count (:profiles payload))))))
+
+(defn reprocess-day!
+ "Manual entrypoint for reprocessing any UTC day without advancing the scheduled watermark."
+ [bucket-date]
+ (process-day! bucket-date {:advance-watermark? false}))
+
+(defn delete-expired-rollups!
+ "Delete rollup rows older than the retention cutoff and return that cutoff day."
+ [retention-days today]
+ (let [retention-days (max 1 (or retention-days 1))
+ cutoff-day (t/minus today (t/days retention-days))]
+ (doseq [model rollup-models]
+ (t2/delete! model :bucket_date [:< cutoff-day]))
+ cutoff-day))
+
+(defn run-batch!
+ "Process all currently targetable closed UTC days and return a run summary."
+ ([] (run-batch! {:today (today-utc)
+ :last-completed-day (parse-watermark (usage-metadata.settings/usage-metadata-last-completed-day))
+ :retention-days (usage-metadata.settings/usage-metadata-retention-days)}))
+ ([{:keys [today last-completed-day retention-days]
+ :or {today (today-utc)}}]
+ (let [days (target-days {:last-completed-day last-completed-day
+ :retention-days retention-days
+ :yesterday (t/minus today (t/days 1))})
+ started-ns (System/nanoTime)
+ cutoff (delete-expired-rollups! retention-days today)
+ initial {:status :success
+ :days-targeted (count days)
+ :days-processed 0
+ :query-execution-rows 0
+ :joined-rows 0
+ :segment-tuples 0
+ :metric-tuples 0
+ :dimension-tuples 0
+ :profile-observations 0
+ :segment-rollup-rows 0
+ :metric-rollup-rows 0
+ :dimension-rollup-rows 0
+ :skipped-rows {}
+ :retention-cutoff cutoff
+ :watermark-advanced-to last-completed-day}]
+ (lib-be/with-metadata-provider-cache
+ (let [raw-result (reduce
+ (fn [summary bucket-date]
+ (try
+ (let [day-result (process-day! bucket-date {:advance-watermark? true})]
+ (-> summary
+ (update :days-processed inc)
+ (update :query-execution-rows + (:query-execution-rows day-result))
+ (update :joined-rows + (:joined-rows day-result))
+ (update :segment-tuples + (:segment-tuples day-result))
+ (update :metric-tuples + (:metric-tuples day-result))
+ (update :dimension-tuples + (:dimension-tuples day-result))
+ (update :profile-observations + (:profile-observations day-result))
+ (update :segment-rollup-rows + (:segment-rollup-rows day-result))
+ (update :metric-rollup-rows + (:metric-rollup-rows day-result))
+ (update :dimension-rollup-rows + (:dimension-rollup-rows day-result))
+ (update :skipped-rows #(merge-with + % (:skipped-rows day-result)))
+ (assoc :watermark-advanced-to (:bucket-date day-result))))
+ (catch Throwable e
+ (reduced
+ (assoc summary
+ :status :failed
+ :error e
+ :watermark-advanced-to
+ (parse-watermark (usage-metadata.settings/usage-metadata-last-completed-day)))))))
+ initial
+ days)
+ elapsed-ms (long (/ (- (System/nanoTime) started-ns) 1000000))
+ final-result (assoc raw-result :elapsed-ms elapsed-ms)]
+ (log/info "Usage metadata batch summary" (dissoc final-result :error))
+ (when-let [e (:error final-result)]
+ (throw e))
+ final-result)))))
diff --git a/src/metabase/usage_metadata/core.clj b/src/metabase/usage_metadata/core.clj
new file mode 100644
index 00000000000..e2a0192a22c
--- /dev/null
+++ b/src/metabase/usage_metadata/core.clj
@@ -0,0 +1,41 @@
+(ns metabase.usage-metadata.core
+ "Public API for usage-metadata rollups."
+ (:require
+ [metabase.usage-metadata.insights :as insights]
+ [metabase.usage-metadata.schema :as usage-metadata.schema]
+ [metabase.util.malli :as mu]))
+
+(set! *warn-on-reflection* true)
+
+(mu/defn implicit-segments :- [:sequential ::usage-metadata.schema/implicit-segment]
+ "Filter predicates users have run ad-hoc that aren't already saved as Segments — surface candidates
+ for promotion to first-class Segments."
+ [opts :- ::usage-metadata.schema/opts]
+ (insights/implicit-segments opts))
+
+(mu/defn implicit-metrics :- [:sequential ::usage-metadata.schema/implicit-metric]
+ "Aggregation patterns users have run ad-hoc that aren't already saved as Metric cards — surface
+ candidates for promotion to first-class Metrics."
+ [opts :- ::usage-metadata.schema/opts]
+ (insights/implicit-metrics opts))
+
+(mu/defn implicit-dimensions :- [:sequential ::usage-metadata.schema/implicit-dimension]
+ "Columns users have grouped by (breakouts) across the window — surface candidates for promotion
+ to first-class dimensions or pre-aggregations."
+ [opts :- ::usage-metadata.schema/opts]
+ (insights/implicit-dimensions opts))
+
+(mu/defn suggested-segments :- [:sequential ::usage-metadata.schema/suggested-segment]
+ "Composite (`:and`) segment definitions that recur across a source's query history. Mined via
+ Apriori FIM over composite rollup baskets: each basket is the atom-set of one stage's top-level
+ `:and`. Itemsets whose atom-set matches a saved Segment's definition are filtered out, so the
+ results are genuinely ad-hoc."
+ [opts :- ::usage-metadata.schema/opts]
+ (insights/suggested-segments-for-owner opts))
+
+(mu/defn profile-observations :- [:sequential ::usage-metadata.schema/profile-observation]
+ "Profile observations recorded for dimensions surfaced by usage-metadata — `:single-value`,
+ `:all-null`, `:low-cardinality` — useful for spotting low-value columns or columns whose
+ cardinality makes them suitable as facets."
+ [opts :- ::usage-metadata.schema/opts]
+ (insights/profile-observations opts))
diff --git a/src/metabase/usage_metadata/extract.clj b/src/metabase/usage_metadata/extract.clj
new file mode 100644
index 00000000000..77858d8ca1a
--- /dev/null
+++ b/src/metabase/usage_metadata/extract.clj
@@ -0,0 +1,303 @@
+(ns metabase.usage-metadata.extract
+ "Extract usage facts (dimensions, segments, metrics) from a normalized MBQL query.
+
+ Every fact is keyed on `(source-type, source-id, ownership-mode)`, where the source
+ is whatever the user was actually querying — a Card or a Table. We don't drill
+ through a card to its underlying table; if you query `card_42` (backed by
+ `table_7`), aggregations over card columns are attributed to the card, not to
+ `table_7`.
+
+ `ownership-mode` is one of:
+
+ - `:direct` — all field-refs in the clause share one owner; one row per
+ field, attributed to that owner.
+ - `:mixed` — field-refs span multiple owners (joins, cross-source
+ expressions). One synthetic row with `source-type`/`source-id`
+ both nil, capturing the whole-clause count.
+ - `:projected` — the per-participant shadow of a `:mixed` row: for each
+ distinct owner involved, emit a row attributing the
+ multi-source clause to that owner. Not a card→table
+ back-projection."
+ (:require
+ [metabase.lib.core :as lib]
+ [metabase.lib.schema.common :as lib.schema.common]
+ [metabase.lib.schema.util :as lib.schema.util]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(defn- canonicalize-for-storage
+ [x]
+ (-> x
+ lib.schema.util/remove-lib-uuids
+ (lib.schema.util/sorted-maps lib.schema.common/unfussy-sorted-map)
+ json/encode))
+
+(defn query-source-table-or-card
+ "Returns the source of a query (table or card ID), or `nil` if none."
+ [query]
+ (when (and query (not (lib/any-native-stage? query)))
+ (or (when-let [table-id (lib/primary-source-table-id query)]
+ {:source-type :table
+ :source-id table-id})
+ (when-let [card-id (lib/primary-source-card-id query)]
+ {:source-type :card
+ :source-id card-id}))))
+
+(defn- owner-from-metadata
+ [root-owner metadata]
+ (cond
+ (and (= :card (:source-type root-owner))
+ (contains? #{:source/card :source/previous-stage :source/aggregations}
+ (:lib/source metadata)))
+ root-owner
+
+ (pos-int? (:lib/card-id metadata))
+ {:source-type :card
+ :source-id (:lib/card-id metadata)}
+
+ (pos-int? (:table-id metadata))
+ {:source-type :table
+ :source-id (:table-id metadata)}
+
+ :else
+ nil))
+
+(defn- referenced-columns-safe
+ [query stage-number clause]
+ (try
+ (lib/referenced-columns query stage-number clause)
+ (catch Throwable e
+ (log/debugf e "usage-metadata: referenced-columns failed (stage %s)" stage-number)
+ nil)))
+
+(defn- field-participants-of-clause
+ "For one MBQL clause, returns distinct `{:field-id ... :owner ...}` maps for each referenced
+ column whose `:id` resolves to a real Field and has a derivable owner."
+ [query stage-number root-owner clause]
+ (into []
+ (comp (keep (fn [col]
+ (when-let [field-id (:id col)]
+ (when (pos-int? field-id)
+ (when-let [owner (owner-from-metadata root-owner col)]
+ {:field-id field-id :owner owner})))))
+ (distinct))
+ (referenced-columns-safe query stage-number clause)))
+
+(defn- aggregation-operator-safe
+ [query stage-number aggregation]
+ (try
+ (:operator (lib/expression-parts query stage-number aggregation))
+ (catch Throwable e
+ (log/debugf e "usage-metadata: aggregation operator extraction failed (stage %s)" stage-number)
+ nil)))
+
+(defn- breakout-column-safe
+ [query stage-number breakout]
+ (try
+ (lib/breakout-column query stage-number breakout)
+ (catch Throwable e
+ (log/debugf e "usage-metadata: breakout-column failed (stage %s)" stage-number)
+ nil)))
+
+(defn- segment-facts-for-clause
+ [query stage-number clause]
+ (let [root-owner (query-source-table-or-card query)
+ field-refs (field-participants-of-clause query stage-number root-owner clause)
+ owners (set (map :owner field-refs))
+ predicate (canonicalize-for-storage clause)]
+ (cond
+ (empty? field-refs)
+ []
+
+ (= 1 (count owners))
+ (let [owner (first owners)]
+ (mapv (fn [{:keys [field-id]}]
+ (assoc owner
+ :ownership-mode :direct
+ :field-id field-id
+ :predicate predicate))
+ field-refs))
+
+ :else
+ (vec
+ (concat
+ [{:source-type nil
+ :source-id nil
+ :ownership-mode :mixed
+ :field-id nil
+ :predicate predicate}]
+ (map (fn [{:keys [field-id owner]}]
+ (assoc owner
+ :ownership-mode :projected
+ :field-id field-id
+ :predicate predicate))
+ field-refs))))))
+
+(defn- segment-facts-for-stage
+ [query stage-number]
+ (into []
+ (mapcat (partial segment-facts-for-clause query stage-number))
+ (or (lib/filters query stage-number) [])))
+
+;; Allowlist of primitive aggregation operators we will record. Sourced from
+;; `metabase.lib.schema.aggregation` (see `::aggregation-clause-tag`).
+;; Composite aggregations (`:-`, `:+`, `:case`, etc.) and references to
+;; saved Metrics (`:metric`, `:measure`, `:aggregation`) are intentionally
+;; excluded — for the implicit-metric discovery use case those are either
+;; already explicit-by-construction or out of scope.
+(def ^:private primitive-aggregation-operators
+ #{:avg
+ :count
+ :cum-count
+ :count-where
+ :distinct
+ :distinct-where
+ :max
+ :median
+ :min
+ :offset
+ :percentile
+ :share
+ :stddev
+ :sum
+ :cum-sum
+ :sum-where
+ :var})
+
+(defn- composite-facts-for-stage
+ "Emit at most one composite fact per stage, treating the stage's top-level filter list as a single
+ implicit-`:and` basket. `lib/atomic-filters` flattens any explicit `:and` children so the basket's
+ atom membership matches the atom rollup's per-atom facts. Stages with fewer than two atoms are
+ skipped (the atom rollup already captures single predicates)."
+ [query stage-number]
+ (let [atoms (or (lib/atomic-filters query stage-number) [])
+ atom-count (count atoms)]
+ (when (>= atom-count 2)
+ (let [root-owner (query-source-table-or-card query)
+ synthetic-and (apply lib/and atoms)
+ canonical-clause (canonicalize-for-storage synthetic-and)
+ canonical-atoms (vec (sort (map canonicalize-for-storage atoms)))
+ field-refs (into []
+ (comp (mapcat (partial field-participants-of-clause query stage-number root-owner))
+ (distinct))
+ atoms)
+ owners (set (map :owner field-refs))
+ base {:clause canonical-clause
+ :atom-fingerprints canonical-atoms
+ :atom-count atom-count}]
+ (cond
+ (empty? field-refs)
+ []
+
+ (= 1 (count owners))
+ [(merge (first owners) base {:ownership-mode :direct})]
+
+ :else
+ (into [(merge base {:source-type nil :source-id nil :ownership-mode :mixed})]
+ (map (fn [owner] (merge owner base {:ownership-mode :projected})))
+ owners))))))
+
+(defn- metric-bases
+ [query stage-number aggregation]
+ (let [root-owner (query-source-table-or-card query)
+ agg-type (aggregation-operator-safe query stage-number aggregation)
+ field-refs (field-participants-of-clause query stage-number root-owner aggregation)
+ owners (set (map :owner field-refs))]
+ (cond
+ ;; Skip composite aggregations and saved-metric references — only record
+ ;; primitive aggregations like sum/count/avg/etc.
+ (not (contains? primitive-aggregation-operators agg-type))
+ []
+
+ (empty? field-refs)
+ [{:source-type (:source-type root-owner)
+ :source-id (:source-id root-owner)
+ :ownership-mode :direct
+ :agg agg-type
+ :agg-field-id nil}]
+
+ (= 1 (count owners))
+ (mapv (fn [{:keys [field-id owner]}]
+ (assoc owner
+ :ownership-mode :direct
+ :agg agg-type
+ :agg-field-id field-id))
+ field-refs)
+
+ :else
+ (vec
+ (concat
+ [{:source-type nil
+ :source-id nil
+ :ownership-mode :mixed
+ :agg agg-type
+ :agg-field-id nil}]
+ (map (fn [{:keys [field-id owner]}]
+ (assoc owner
+ :ownership-mode :projected
+ :agg agg-type
+ :agg-field-id field-id))
+ field-refs))))))
+
+(defn- temporal-breakout-from-column
+ [breakout-col]
+ (when-let [unit (lib/raw-temporal-bucket breakout-col)]
+ (when-let [field-id (:id breakout-col)]
+ (when (pos-int? field-id)
+ {:temporal-field-id field-id
+ :temporal-unit unit}))))
+
+(defn- metric-facts-for-stage
+ [query stage-number]
+ (let [breakout-cols (mapv (partial breakout-column-safe query stage-number)
+ (or (lib/breakouts query stage-number) []))
+ temporal-breakouts (into [] (keep temporal-breakout-from-column) breakout-cols)]
+ (into []
+ (mapcat (fn [aggregation]
+ (for [base (metric-bases query stage-number aggregation)
+ temporal (if (seq temporal-breakouts)
+ temporal-breakouts
+ [{:temporal-field-id nil
+ :temporal-unit nil}])]
+ (merge base temporal))))
+ (or (lib/aggregations query stage-number) []))))
+
+(defn- serialize-binning
+ [binning]
+ (canonicalize-for-storage
+ (cond-> {:strategy (:strategy binning)}
+ (:num-bins binning) (assoc :num-bins (:num-bins binning))
+ (:bin-width binning) (assoc :bin-width (:bin-width binning)))))
+
+(defn- dimension-facts-for-stage
+ [query stage-number]
+ (let [root-owner (query-source-table-or-card query)]
+ (into []
+ (keep (fn [breakout]
+ (when-let [col (breakout-column-safe query stage-number breakout)]
+ (when-let [field-id (:id col)]
+ (when (pos-int? field-id)
+ (when-let [owner (owner-from-metadata root-owner col)]
+ (assoc owner
+ :ownership-mode :direct
+ :field-id field-id
+ :temporal-unit (lib/raw-temporal-bucket col)
+ :binning (some-> (lib/binning col) serialize-binning))))))))
+ (or (lib/breakouts query stage-number) []))))
+
+(defn extract-usage-facts
+ "Extract fact-level usage tuples from a normalized MBQL query."
+ [query]
+ (reduce
+ (fn [{:keys [segments composites metrics dimensions]} stage-number]
+ {:segments (into segments (segment-facts-for-stage query stage-number))
+ :composites (into composites (composite-facts-for-stage query stage-number))
+ :metrics (into metrics (metric-facts-for-stage query stage-number))
+ :dimensions (into dimensions (dimension-facts-for-stage query stage-number))})
+ {:segments []
+ :composites []
+ :metrics []
+ :dimensions []}
+ (range (lib/stage-count query))))
diff --git a/src/metabase/usage_metadata/init.clj b/src/metabase/usage_metadata/init.clj
new file mode 100644
index 00000000000..5211ed6b3b1
--- /dev/null
+++ b/src/metabase/usage_metadata/init.clj
@@ -0,0 +1,4 @@
+(ns metabase.usage-metadata.init
+ (:require
+ [metabase.usage-metadata.settings]
+ [metabase.usage-metadata.task.process]))
diff --git a/src/metabase/usage_metadata/insights.clj b/src/metabase/usage_metadata/insights.clj
new file mode 100644
index 00000000000..fda4f67ab63
--- /dev/null
+++ b/src/metabase/usage_metadata/insights.clj
@@ -0,0 +1,579 @@
+(ns metabase.usage-metadata.insights
+ "Read-side helpers over usage-metadata rollups — consumer of the batch pipeline."
+ (:require
+ [clojure.core.memoize :as memoize]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.usage-metadata.extract :as usage-metadata.extract]
+ [metabase.usage-metadata.models.source-dimension-daily]
+ [metabase.usage-metadata.models.source-dimension-profile-daily]
+ [metabase.usage-metadata.models.source-metric-daily]
+ [metabase.usage-metadata.models.source-segment-composite-daily]
+ [metabase.usage-metadata.models.source-segment-daily]
+ [metabase.usage-metadata.schema :as usage-metadata.schema]
+ [metabase.util.json :as json]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn- decode-predicate
+ "Parse a canonicalized predicate JSON string back into an MBQL 5 clause."
+ [predicate-json]
+ (some-> predicate-json
+ json/decode+kw
+ lib/normalize))
+
+(defn- decode-binning
+ "Parse a canonicalized binning JSON string back into a map."
+ [binning-json]
+ (some-> binning-json
+ json/decode+kw))
+
+(defn- build-field-index
+ "Bulk-fetch Field rows for `field-ids` and return a map of `id -> {:id :name :display-name}`.
+
+ Avoids the per-field `Field` + `Table` + metadata-provider roundtrip that the single-row
+ helper used to do."
+ [field-ids]
+ (let [field-ids (into #{} (filter pos-int?) field-ids)
+ rows (when (seq field-ids)
+ (t2/select [:model/Field :id :name :display_name] :id [:in field-ids]))]
+ (into {}
+ (map (fn [{:keys [id name display_name]}]
+ [id {:id id
+ :name name
+ :display-name (or display_name name)}]))
+ rows)))
+
+(defn- build-source-index
+ "Bulk-fetch Table and Card rows for the `[source-type source-id]` tuples present in
+ `source-keys` and return a map keyed by `[source-type source-id]`."
+ [source-keys]
+ (let [by-type (group-by first source-keys)
+ table-ids (into #{} (comp (keep second) (filter pos-int?)) (get by-type :table))
+ card-ids (into #{} (comp (keep second) (filter pos-int?)) (get by-type :card))
+ tables (when (seq table-ids)
+ (t2/select [:model/Table :id :name :display_name :db_id :schema]
+ :id [:in table-ids]))
+ cards (when (seq card-ids)
+ (t2/select [:model/Card :id :name] :id [:in card-ids]))]
+ (into {}
+ cat
+ [(map (fn [{:keys [id name display_name db_id schema]}]
+ [[:table id] {:type :table
+ :id id
+ :db-id db_id
+ :schema schema
+ :name name
+ :display-name (or display_name name)}])
+ tables)
+ (map (fn [{:keys [id name]}]
+ [[:card id] {:type :card
+ :id id
+ :name name
+ :display-name name}])
+ cards)])))
+
+(defn- predicate-field-ids
+ "Return distinct field ids referenced in a decoded predicate."
+ [predicate]
+ (when predicate
+ (try
+ (vec (lib/all-field-ids predicate))
+ (catch Throwable e
+ (log/debug e "usage-metadata: predicate-field-ids failed")
+ []))))
+
+;; No SQL LIMIT here: callers filter saved-signature collisions in Clojure, so a SQL LIMIT can under-deliver.
+(defn- grouped-segment-rows
+ "Group + sum `source_segment_daily` counts for a source filter."
+ [{:keys [source-type source-id bucket-start bucket-end]}]
+ (let [where (cond-> [:and
+ [:in :ownership_mode ["direct" "projected"]]]
+ source-type (conj [:= :source_type (name source-type)])
+ source-id (conj [:= :source_id source-id])
+ bucket-start (conj [:>= :bucket_date bucket-start])
+ bucket-end (conj [:<= :bucket_date bucket-end]))]
+ (t2/select [:model/SourceSegmentDaily
+ :source_type
+ :source_id
+ :field_id
+ :predicate
+ [[:sum :count] :total_count]]
+ {:where where
+ :group-by [:source_type :source_id :field_id :predicate]
+ :order-by [[:total_count :desc]]})))
+
+(defn- grouped-metric-rows
+ "Group + sum `source_metric_daily` counts for a source filter."
+ [{:keys [source-type source-id bucket-start bucket-end]}]
+ (let [where (cond-> [:and
+ [:in :ownership_mode ["direct" "projected"]]]
+ source-type (conj [:= :source_type (name source-type)])
+ source-id (conj [:= :source_id source-id])
+ bucket-start (conj [:>= :bucket_date bucket-start])
+ bucket-end (conj [:<= :bucket_date bucket-end]))]
+ (t2/select [:model/SourceMetricDaily
+ :source_type
+ :source_id
+ :agg_type
+ :agg_field_id
+ :temporal_field_id
+ :temporal_unit
+ [[:sum :count] :total_count]]
+ {:where where
+ :group-by [:source_type :source_id :agg_type :agg_field_id :temporal_field_id :temporal_unit]
+ :order-by [[:total_count :desc]]})))
+
+(defn- grouped-dimension-rows
+ "Group + sum `source_dimension_daily` counts for a source filter."
+ [{:keys [source-type source-id bucket-start bucket-end]}]
+ (let [where (cond-> [:and
+ [:in :ownership_mode ["direct" "projected"]]]
+ source-type (conj [:= :source_type (name source-type)])
+ source-id (conj [:= :source_id source-id])
+ bucket-start (conj [:>= :bucket_date bucket-start])
+ bucket-end (conj [:<= :bucket_date bucket-end]))]
+ (t2/select [:model/SourceDimensionDaily
+ :source_type
+ :source_id
+ :field_id
+ :temporal_unit
+ :binning
+ [[:sum :count] :total_count]]
+ {:where where
+ :group-by [:source_type :source_id :field_id :temporal_unit :binning]
+ :order-by [[:total_count :desc]]})))
+
+(defn- decode-atom-fingerprints [x]
+ (cond
+ (sequential? x) (vec x)
+ (string? x) (vec (json/decode x))
+ :else []))
+
+(defn- grouped-composite-rows
+ "Group + sum `source_segment_composite_daily` counts for a source filter.
+
+ Each returned row carries the whole-clause JSON, the atom-fingerprint array, and the summed count
+ across the window — the input shape expected by the FIM pass."
+ [{:keys [source-type source-id bucket-start bucket-end]}]
+ (let [where (cond-> [:and
+ [:in :ownership_mode ["direct" "projected"]]]
+ source-type (conj [:= :source_type (name source-type)])
+ source-id (conj [:= :source_id source-id])
+ bucket-start (conj [:>= :bucket_date bucket-start])
+ bucket-end (conj [:<= :bucket_date bucket-end]))]
+ (->> (t2/select [:model/SourceSegmentCompositeDaily
+ :source_type
+ :source_id
+ :clause
+ :atom_fingerprints
+ :atom_count
+ [[:sum :count] :total_count]]
+ {:where where
+ :group-by [:source_type :source_id :clause :atom_fingerprints :atom_count]
+ :order-by [[:total_count :desc]]})
+ (mapv (fn [row]
+ (update row :atom_fingerprints decode-atom-fingerprints))))))
+
+(defn- grouped-profile-rows
+ "Group + sum `source_dimension_profile_daily` counts for a source filter."
+ [{:keys [source-type source-id bucket-start bucket-end]}]
+ (let [where (cond-> [:and]
+ source-type (conj [:= :source_type (name source-type)])
+ source-id (conj [:= :source_id source-id])
+ bucket-start (conj [:>= :bucket_date bucket-start])
+ bucket-end (conj [:<= :bucket_date bucket-end]))]
+ (t2/select [:model/SourceDimensionProfileDaily
+ :source_type
+ :source_id
+ :field_id
+ :source_basis
+ :observation_type
+ :observation_value
+ [[:sum :count] :total_count]]
+ {:where where
+ :group-by [:source_type :source_id :field_id :source_basis :observation_type :observation_value]
+ :order-by [[:total_count :desc]]})))
+
+(defn- wrap-query
+ "Wrap a raw MBQL map in a full lib query using the app DB metadata-provider. Returns nil on failure."
+ [database-id query-map]
+ (when (and (pos-int? database-id) (seq query-map))
+ (try
+ (lib/query (lib-be/application-database-metadata-provider database-id) query-map)
+ (catch Throwable e
+ (log/debug e "Failed to wrap query for usage-metadata insights")
+ nil))))
+
+(defn- extract-facts
+ [database-id query-map]
+ (when-let [q (wrap-query database-id query-map)]
+ (try
+ (usage-metadata.extract/extract-usage-facts q)
+ (catch Throwable e
+ (log/debug e "Failed to extract usage facts for usage-metadata insights")
+ nil))))
+
+(def ^:private cache-ttl-ms
+ (* 60 1000))
+
+(defn- existing-segment-predicates*
+ [[source-type source-id]]
+ (let [where (cond-> [:and [:= :archived false]]
+ (and (= source-type :table) source-id) (conj [:= :table_id source-id]))
+ segments (t2/select [:model/Segment :id :table_id :definition] {:where where})
+ table-ids (into #{} (comp (keep :table_id) (filter pos-int?)) segments)
+ table->db (when (seq table-ids)
+ (into {}
+ (map (juxt :id :db_id))
+ (t2/select [:model/Table :id :db_id] :id [:in table-ids])))]
+ (lib-be/with-metadata-provider-cache
+ (into #{}
+ (mapcat (fn [{:keys [table_id definition]}]
+ (when (and (pos-int? table_id) (seq definition))
+ (when-let [db-id (get table->db table_id)]
+ (let [facts (:segments (extract-facts db-id definition))]
+ (for [{:keys [predicate]} facts
+ :when predicate]
+ [:table table_id predicate]))))))
+ segments))))
+
+(def ^:private existing-segment-predicates*-memo
+ (memoize/ttl existing-segment-predicates* :ttl/threshold cache-ttl-ms))
+
+(defn- existing-segment-predicates
+ "Set of `[source-type source-id predicate-json]` tuples for non-archived Segments whose
+ atomic filter clauses would collide with stored implicit segment predicates."
+ [{:keys [source-type source-id]}]
+ (existing-segment-predicates*-memo [source-type source-id]))
+
+(defn- existing-metric-signatures*
+ []
+ (let [cards (t2/select [:model/Card :id :database_id :dataset_query :card_schema]
+ :type "metric"
+ :archived false)]
+ (lib-be/with-metadata-provider-cache
+ (into #{}
+ (mapcat (fn [{:keys [database_id dataset_query]}]
+ (when (seq dataset_query)
+ (let [facts (:metrics (extract-facts database_id dataset_query))]
+ (for [{:keys [source-type source-id ownership-mode
+ agg agg-field-id temporal-field-id temporal-unit]} facts
+ ;; only :direct signatures are comparable to stored rollup rows for this source
+ :when (and source-type (= ownership-mode :direct))]
+ [source-type source-id agg agg-field-id temporal-field-id temporal-unit])))))
+ cards))))
+
+(def ^:private existing-metric-signatures*-memo
+ (memoize/ttl (fn [_] (existing-metric-signatures*)) :ttl/threshold cache-ttl-ms))
+
+(defn- existing-metric-signatures
+ "Set of `[source-type source-id agg-type agg-field-id temporal-field-id temporal-unit]`
+ tuples for non-archived Metric cards (Cards of type `metric`). Scan is independent of opts;
+ TTL-memoized with a singleton key."
+ []
+ (existing-metric-signatures*-memo ::all))
+
+(mu/defn implicit-segments :- [:sequential ::usage-metadata.schema/implicit-segment]
+ "Top implicit segments recorded across usage-metadata rollups.
+
+ Predicates that correspond to *existing* saved Segments are filtered out — only truly
+ ad-hoc filter patterns are returned."
+ ([] (implicit-segments {}))
+ ([{:keys [limit] :or {limit 10} :as opts} :- ::usage-metadata.schema/opts]
+ (let [existing (existing-segment-predicates opts)
+ raw-rows (remove (fn [{:keys [source_type source_id predicate]}]
+ (contains? existing [source_type source_id predicate]))
+ (grouped-segment-rows opts))
+ ;; Decode each predicate + its field-ids in one pass so we can bulk-fetch below.
+ enriched (mapv (fn [{:keys [predicate] :as row}]
+ (let [decoded (decode-predicate predicate)]
+ (assoc row
+ ::decoded decoded
+ ::field-ids (predicate-field-ids decoded))))
+ raw-rows)
+ source-idx (build-source-index
+ (into #{} (map (juxt :source_type :source_id)) enriched))
+ field-idx (build-field-index
+ (into #{} (mapcat ::field-ids) enriched))]
+ (into []
+ (comp
+ (keep (fn [row]
+ (when-let [source (source-idx [(:source_type row) (:source_id row)])]
+ (let [fields (into [] (keep field-idx) (::field-ids row))]
+ (when (seq fields)
+ {:predicate (::decoded row)
+ :source source
+ :fields fields
+ :count (long (:total_count row))})))))
+ (take limit))
+ enriched))))
+
+(mu/defn implicit-metrics :- [:sequential ::usage-metadata.schema/implicit-metric]
+ "Top implicit metrics recorded across usage-metadata rollups.
+
+ Aggregations that correspond to *existing* saved Metrics (Cards of type `metric`) are
+ filtered out — only truly ad-hoc aggregation patterns are returned."
+ ([] (implicit-metrics {}))
+ ([{:keys [limit] :or {limit 10} :as opts} :- ::usage-metadata.schema/opts]
+ (let [existing (existing-metric-signatures)
+ rows (remove (fn [{:keys [source_type source_id agg_type agg_field_id temporal_field_id temporal_unit]}]
+ (contains? existing [source_type source_id agg_type agg_field_id temporal_field_id temporal_unit]))
+ (grouped-metric-rows opts))
+ source-idx (build-source-index
+ (into #{} (map (juxt :source_type :source_id)) rows))
+ field-idx (build-field-index
+ (into #{} (mapcat (juxt :agg_field_id :temporal_field_id)) rows))]
+ (into []
+ (comp
+ (keep (fn [{:keys [source_type source_id agg_type agg_field_id temporal_field_id temporal_unit total_count]}]
+ (when-let [source (source-idx [source_type source_id])]
+ {:source source
+ :aggregation {:type agg_type
+ :field (field-idx agg_field_id)
+ :temporal-field (field-idx temporal_field_id)
+ :temporal-unit temporal_unit}
+ :count (long total_count)})))
+ (take limit))
+ rows))))
+
+(mu/defn implicit-dimensions :- [:sequential ::usage-metadata.schema/implicit-dimension]
+ "Top implicit dimensions recorded across usage-metadata rollups."
+ ([] (implicit-dimensions {}))
+ ([{:keys [limit] :or {limit 10} :as opts} :- ::usage-metadata.schema/opts]
+ (let [rows (grouped-dimension-rows opts)
+ source-idx (build-source-index
+ (into #{} (map (juxt :source_type :source_id)) rows))
+ field-idx (build-field-index
+ (into #{} (keep :field_id) rows))]
+ (into []
+ (comp
+ (keep (fn [{:keys [source_type source_id field_id temporal_unit binning total_count]}]
+ (when-let [source (source-idx [source_type source_id])]
+ (when-let [field (field-idx field_id)]
+ {:source source
+ :dimension {:field field
+ :temporal-unit temporal_unit
+ :binning (decode-binning binning)}
+ :count (long total_count)}))))
+ (take limit))
+ rows))))
+
+(def ^:private fim-absolute-support-floor 2)
+(def ^:private fim-relative-support-floor 0.2)
+(def ^:private fim-k-min 2)
+(def ^:private fim-k-max 5)
+(def ^:private fim-default-limit 20)
+
+(defn- rows->baskets
+ "Project rollup rows to `{:atoms #{...} :count n}` baskets for mining.
+
+ Coerces `total_count` to `long` so downstream FIM math (and the `:support` field) stays
+ integer — MariaDB/MySQL return `SUM(int_col)` as `BigDecimal`."
+ [rows]
+ (into []
+ (keep (fn [{:keys [atom_fingerprints total_count]}]
+ (when (>= (count atom_fingerprints) fim-k-min)
+ {:atoms (set atom_fingerprints)
+ :count (long total_count)})))
+ rows))
+
+(defn- itemset-support
+ [baskets itemset]
+ (reduce (fn [acc {:keys [atoms count]}]
+ (if (every? atoms itemset)
+ (+ acc count)
+ acc))
+ 0
+ baskets))
+
+(defn- any-atom-support
+ "Number of baskets containing at least one atom of `itemset` (denominator for the relative support check)."
+ [baskets itemset]
+ (reduce (fn [acc {:keys [atoms count]}]
+ (if (some atoms itemset)
+ (+ acc count)
+ acc))
+ 0
+ baskets))
+
+(defn- frequent-singletons
+ [baskets absolute-floor]
+ (let [counts (reduce (fn [m {:keys [atoms count]}]
+ (reduce (fn [m a] (update m a (fnil + 0) count)) m atoms))
+ {}
+ baskets)]
+ (into {}
+ (filter (fn [[_ n]] (>= n absolute-floor)))
+ counts)))
+
+(defn- apriori-join
+ "Generate k+1 candidate itemsets by joining k-itemsets sharing a k-1 prefix."
+ [lk-vecs]
+ (let [by-prefix (group-by (fn [is] (subvec is 0 (dec (count is)))) lk-vecs)]
+ (into #{}
+ (mapcat (fn [group]
+ (for [a group
+ b group
+ :when (neg? (compare (peek a) (peek b)))]
+ (conj a (peek b)))))
+ (vals by-prefix))))
+
+(defn- has-all-k-subsets?
+ [lk-set candidate]
+ (let [n (count candidate)]
+ (every? (fn [i]
+ (let [sub (into (subvec candidate 0 i) (subvec candidate (inc i)))]
+ (contains? lk-set sub)))
+ (range n))))
+
+(defn- mine-itemsets
+ "Apriori up to size `fim-k-max`. Returns `{itemset-vec support}` for itemsets of size ≥ `fim-k-min`."
+ [baskets]
+ (let [singletons (frequent-singletons baskets fim-absolute-support-floor)
+ l1-vecs (vec (sort (map vector (keys singletons))))]
+ (loop [lk l1-vecs
+ k 1
+ acc {}]
+ (if (or (empty? lk) (>= k fim-k-max))
+ acc
+ (let [lk-set (set lk)
+ candidates (apriori-join lk)
+ pruned (into [] (filter (partial has-all-k-subsets? lk-set)) candidates)
+ counted (into {}
+ (keep (fn [c]
+ (let [s (itemset-support baskets (set c))]
+ (when (>= s fim-absolute-support-floor) [c s]))))
+ pruned)
+ next-k (inc k)
+ acc (if (>= next-k fim-k-min)
+ (merge acc counted)
+ acc)]
+ (recur (vec (sort (keys counted))) next-k acc))))))
+
+(defn- closed-only
+ "Keep only itemsets that have no proper superset of equal support — the closed frequent itemsets."
+ [itemset->support]
+ (let [entries (vec itemset->support)]
+ (into {}
+ (remove (fn [[is sup]]
+ (let [is-set (set is)
+ is-n (count is)]
+ (some (fn [[other other-sup]]
+ (and (= sup other-sup)
+ (> (count other) is-n)
+ (every? (set other) is-set)))
+ entries))))
+ entries)))
+
+(defn- relative-support-ok?
+ [baskets itemset support]
+ (let [denom (any-atom-support baskets itemset)]
+ (or (zero? denom)
+ (>= (/ support (double denom)) fim-relative-support-floor))))
+
+(defn- rebuild-and-clause
+ [fingerprints]
+ (let [atoms (into []
+ (keep decode-predicate)
+ fingerprints)]
+ (when (>= (count atoms) fim-k-min)
+ (lib/simplify-compound-filter (apply lib/and atoms)))))
+
+(defn- existing-composite-atomsets*
+ [[source-type source-id]]
+ (let [where (cond-> [:and [:= :archived false]]
+ (and (= source-type :table) source-id) (conj [:= :table_id source-id]))
+ segments (t2/select [:model/Segment :id :table_id :definition] {:where where})
+ table-ids (into #{} (comp (keep :table_id) (filter pos-int?)) segments)
+ table->db (when (seq table-ids)
+ (into {}
+ (map (juxt :id :db_id))
+ (t2/select [:model/Table :id :db_id] :id [:in table-ids])))]
+ (lib-be/with-metadata-provider-cache
+ (into #{}
+ (mapcat (fn [{:keys [table_id definition]}]
+ (when (and (pos-int? table_id) (seq definition))
+ (when-let [db-id (get table->db table_id)]
+ (let [facts (:composites (extract-facts db-id definition))]
+ (for [{:keys [atom-fingerprints]} facts
+ :when (>= (count atom-fingerprints) fim-k-min)]
+ [:table table_id (set atom-fingerprints)]))))))
+ segments))))
+
+(def ^:private existing-composite-atomsets*-memo
+ (memoize/ttl existing-composite-atomsets* :ttl/threshold cache-ttl-ms))
+
+(defn- existing-composite-atomsets
+ "Set of `[source-type source-id #{atom-fingerprint ...}]` tuples for non-archived Segments whose
+ definitions are whole-:and baskets. Used to filter out suggestions that already exist as saved Segments."
+ [{:keys [source-type source-id]}]
+ (existing-composite-atomsets*-memo [source-type source-id]))
+
+(mu/defn suggested-segments-for-owner :- [:sequential ::usage-metadata.schema/suggested-segment]
+ "Suggest composite (`:and`) segment definitions that recur across a source's query history but
+ have not been saved as Segments yet. Implemented as Apriori FIM over composite rollup baskets:
+ each rollup row is a basket whose items are the atomic predicates of one stage's top-level `:and`.
+ We mine closed frequent itemsets and reconstruct each surviving itemset as an `:and` MBQL clause.
+
+ `:itemset-size` is bounded by `fim-k-min`/`fim-k-max` (2..5). `:support` is the weighted count of
+ baskets containing ALL of the itemset's atoms (basket weight = the rollup row's `:count`).
+ `:support-ratio` is `support / any-atom-support` and is floored by `fim-relative-support-floor`.
+
+ Results are sorted by `:support` desc, then by `:itemset-size` desc — at equal support, larger
+ recurring `:and`s rank higher, since they encode more user intent. Truncated to `:limit`."
+ ([] (suggested-segments-for-owner {}))
+ ([{:keys [limit] :or {limit fim-default-limit} :as opts} :- ::usage-metadata.schema/opts]
+ (let [rows (grouped-composite-rows opts)
+ by-source (group-by (juxt :source_type :source_id) rows)
+ source-idx (build-source-index (keys by-source))
+ candidates (into []
+ (mapcat (fn [[[source-type source-id] source-rows]]
+ (when-let [source (source-idx [source-type source-id])]
+ (let [baskets (rows->baskets source-rows)
+ existing (existing-composite-atomsets {:source-type source-type
+ :source-id source-id})
+ mined (closed-only (mine-itemsets baskets))]
+ (for [[itemset-vec support] mined
+ :let [itemset (set itemset-vec)]
+ :when (and (relative-support-ok? baskets itemset-vec support)
+ (not (contains? existing [source-type source-id itemset])))
+ :let [clause (rebuild-and-clause itemset-vec)
+ denom (any-atom-support baskets itemset-vec)]
+ :when clause]
+ {:clause clause
+ :itemset-size (count itemset-vec)
+ :source source
+ :support support
+ :support-ratio (if (pos? denom) (/ support (double denom)) 0.0)})))))
+ by-source)]
+ (into []
+ (take limit)
+ (sort-by (juxt (comp - :support) (comp - :itemset-size)) candidates)))))
+
+(mu/defn profile-observations :- [:sequential ::usage-metadata.schema/profile-observation]
+ "Top dimension profile observations recorded across usage-metadata rollups."
+ ([] (profile-observations {}))
+ ([{:keys [limit] :or {limit 10} :as opts} :- ::usage-metadata.schema/opts]
+ (let [rows (grouped-profile-rows opts)
+ source-idx (build-source-index
+ (into #{} (map (juxt :source_type :source_id)) rows))
+ field-idx (build-field-index
+ (into #{} (keep :field_id) rows))]
+ (into []
+ (comp
+ (keep (fn [{:keys [source_type source_id field_id source_basis
+ observation_type observation_value total_count]}]
+ (when-let [source (source-idx [source_type source_id])]
+ (when-let [field (field-idx field_id)]
+ {:source source
+ :field field
+ :basis source_basis
+ :observation {:type observation_type
+ :value observation_value}
+ :count (long total_count)}))))
+ (take limit))
+ rows))))
diff --git a/src/metabase/usage_metadata/models/source_dimension_daily.clj b/src/metabase/usage_metadata/models/source_dimension_daily.clj
new file mode 100644
index 00000000000..24eb1f1d1b4
--- /dev/null
+++ b/src/metabase/usage_metadata/models/source_dimension_daily.clj
@@ -0,0 +1,14 @@
+(ns metabase.usage-metadata.models.source-dimension-daily
+ (:require
+ [metabase.models.interface :as mi]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/SourceDimensionDaily [_model] :source_dimension_daily)
+
+(t2/deftransforms :model/SourceDimensionDaily
+ {:source_type mi/transform-keyword
+ :ownership_mode mi/transform-keyword
+ :temporal_unit mi/transform-keyword})
+
+(derive :model/SourceDimensionDaily :metabase/model)
diff --git a/src/metabase/usage_metadata/models/source_dimension_profile_daily.clj b/src/metabase/usage_metadata/models/source_dimension_profile_daily.clj
new file mode 100644
index 00000000000..99b295da627
--- /dev/null
+++ b/src/metabase/usage_metadata/models/source_dimension_profile_daily.clj
@@ -0,0 +1,19 @@
+(ns metabase.usage-metadata.models.source-dimension-profile-daily
+ "Daily rollup of dimension-profile observations (`:single-value`, `:all-null`,
+ `:low-cardinality`) for fields that showed up as direct table dimensions in
+ the day's activity. Each observation is read from the field's existing
+ `Field.fingerprint` (populated by the sync fingerprinter); `source_basis` is
+ `:fingerprint`."
+ (:require
+ [metabase.models.interface :as mi]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/SourceDimensionProfileDaily [_model] :source_dimension_profile_daily)
+
+(t2/deftransforms :model/SourceDimensionProfileDaily
+ {:source_type mi/transform-keyword
+ :source_basis mi/transform-keyword
+ :observation_type mi/transform-keyword})
+
+(derive :model/SourceDimensionProfileDaily :metabase/model)
diff --git a/src/metabase/usage_metadata/models/source_metric_daily.clj b/src/metabase/usage_metadata/models/source_metric_daily.clj
new file mode 100644
index 00000000000..c9af4da537e
--- /dev/null
+++ b/src/metabase/usage_metadata/models/source_metric_daily.clj
@@ -0,0 +1,15 @@
+(ns metabase.usage-metadata.models.source-metric-daily
+ (:require
+ [metabase.models.interface :as mi]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/SourceMetricDaily [_model] :source_metric_daily)
+
+(t2/deftransforms :model/SourceMetricDaily
+ {:source_type mi/transform-keyword
+ :ownership_mode mi/transform-keyword
+ :agg_type mi/transform-keyword
+ :temporal_unit mi/transform-keyword})
+
+(derive :model/SourceMetricDaily :metabase/model)
diff --git a/src/metabase/usage_metadata/models/source_segment_composite_daily.clj b/src/metabase/usage_metadata/models/source_segment_composite_daily.clj
new file mode 100644
index 00000000000..0a64cd378cb
--- /dev/null
+++ b/src/metabase/usage_metadata/models/source_segment_composite_daily.clj
@@ -0,0 +1,13 @@
+(ns metabase.usage-metadata.models.source-segment-composite-daily
+ (:require
+ [metabase.models.interface :as mi]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/SourceSegmentCompositeDaily [_model] :source_segment_composite_daily)
+
+(t2/deftransforms :model/SourceSegmentCompositeDaily
+ {:source_type mi/transform-keyword
+ :ownership_mode mi/transform-keyword})
+
+(derive :model/SourceSegmentCompositeDaily :metabase/model)
diff --git a/src/metabase/usage_metadata/models/source_segment_daily.clj b/src/metabase/usage_metadata/models/source_segment_daily.clj
new file mode 100644
index 00000000000..e386ae3044d
--- /dev/null
+++ b/src/metabase/usage_metadata/models/source_segment_daily.clj
@@ -0,0 +1,13 @@
+(ns metabase.usage-metadata.models.source-segment-daily
+ (:require
+ [metabase.models.interface :as mi]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/SourceSegmentDaily [_model] :source_segment_daily)
+
+(t2/deftransforms :model/SourceSegmentDaily
+ {:source_type mi/transform-keyword
+ :ownership_mode mi/transform-keyword})
+
+(derive :model/SourceSegmentDaily :metabase/model)
diff --git a/src/metabase/usage_metadata/schema.clj b/src/metabase/usage_metadata/schema.clj
new file mode 100644
index 00000000000..569e4042c6f
--- /dev/null
+++ b/src/metabase/usage_metadata/schema.clj
@@ -0,0 +1,112 @@
+(ns metabase.usage-metadata.schema
+ "Shared malli schemas for usage-metadata public API inputs and results.
+
+ These are the shape contracts pinned at the `metabase.usage-metadata.core` boundary and
+ enforced inside `metabase.usage-metadata.insights` producers."
+ (:require
+ [metabase.util.malli.registry :as mr]))
+
+(mr/def ::source-type [:enum :table :card])
+
+(mr/def ::opts
+ [:map {:closed true}
+ [:source-type {:optional true, :description "Restrict to a specific source kind."}
+ [:maybe ::source-type]]
+ [:source-id {:optional true, :description "Restrict to a specific source id."}
+ [:maybe pos-int?]]
+ [:bucket-start {:optional true, :description "Inclusive lower bound on rollup bucket_date."}
+ [:maybe :time/local-date]]
+ [:bucket-end {:optional true, :description "Inclusive upper bound on rollup bucket_date."}
+ [:maybe :time/local-date]]
+ [:limit {:optional true, :description "Maximum number of results to return."}
+ [:maybe pos-int?]]])
+
+(mr/def ::source
+ [:map
+ [:type ::source-type]
+ [:id pos-int?]
+ [:name [:maybe :string]]
+ [:display-name [:maybe :string]]
+ ;; :db-id / :schema are present only on :table sources.
+ [:db-id {:optional true} [:maybe pos-int?]]
+ [:schema {:optional true} [:maybe :string]]])
+
+(mr/def ::field
+ [:map
+ [:id pos-int?]
+ [:name [:maybe :string]]
+ [:display-name [:maybe :string]]])
+
+(mr/def ::mbql-clause
+ [:fn {:error/message "expected an MBQL clause"}
+ (fn [x] (and (vector? x) (keyword? (first x))))])
+
+(mr/def ::implicit-segment
+ [:map {:closed true}
+ [:predicate {:description "The MBQL filter clause as it appeared in queries, decoded from canonical storage."}
+ [:maybe ::mbql-clause]]
+ [:source {:description "The table or card the predicate was applied to."}
+ ::source]
+ [:fields {:description "Field metadata for every field referenced inside the predicate. Rows with no resolvable fields are dropped upstream."}
+ [:sequential {:min 1} ::field]]
+ [:count {:description "Number of query executions in the window whose stage used this exact predicate."}
+ pos-int?]])
+
+(mr/def ::implicit-metric
+ [:map {:closed true}
+ [:source {:description "The table or card the aggregation was applied to."}
+ ::source]
+ [:aggregation [:map {:closed true}
+ [:type {:description "Primitive op (`:sum`, `:count`, `:avg`, …). Composite aggregations and saved-metric refs are excluded upstream."}
+ :keyword]
+ [:field {:description "The aggregated column's metadata (nil for `:count`)."}
+ [:maybe ::field]]
+ [:temporal-field {:description "The temporal breakout column joined to this aggregation, if any. nil means an untimed aggregation."}
+ [:maybe ::field]]
+ [:temporal-unit {:description "Bucket of the temporal breakout (`:day`, `:month`, …)."}
+ [:maybe :keyword]]]]
+ [:count {:description "Number of executions whose stage used this aggregation shape."}
+ pos-int?]])
+
+(mr/def ::implicit-dimension
+ [:map {:closed true}
+ [:source {:description "The table or card the breakout was applied to."}
+ ::source]
+ [:dimension [:map {:closed true}
+ [:field {:description "The broken-out column's metadata."}
+ ::field]
+ [:temporal-unit {:description "Temporal bucket (`:day`, `:month`, …) if the breakout was temporally bucketed; nil otherwise."}
+ [:maybe :keyword]]
+ [:binning {:description "Decoded binning spec (`:num-bins`, `:bin-width`, `:strategy`) if the breakout was binned; nil otherwise."}
+ [:maybe :map]]]]
+ [:count {:description "Number of executions whose stage broke out by this exact shape."}
+ pos-int?]])
+
+(mr/def ::suggested-segment
+ [:map {:closed true}
+ [:clause {:description "Reconstructed `[:and ...]` MBQL clause built from the itemset's atoms — what a caller would offer the user to save as a new Segment."}
+ ::mbql-clause]
+ [:itemset-size {:description "Number of atomic predicates the suggestion combines (`k` in FIM terms; 2..5 bounded by `fim-k-min`/`fim-k-max`). NOT the size of the source baskets the itemset was mined from — those can be larger."}
+ pos-int?]
+ [:source {:description "The table or card the suggestion is attributed to. Suggestions live within a single source; cross-source composites are not mined."}
+ ::source]
+ [:support {:description "Weighted count of baskets containing ALL the itemset's atoms (basket weight is the rollup row's execution count). Primary ranking key."}
+ pos-int?]
+ [:support-ratio {:description "Fraction of baskets-touching-any-atom that also contain the full itemset. Guards against an individually popular atom dragging a co-occurrence that doesn't actually travel together. Floored by `fim-relative-support-floor`."}
+ number?]])
+
+(mr/def ::profile-observation
+ [:map {:closed true}
+ [:source {:description "The table or card the field lives on."}
+ ::source]
+ [:field {:description "Field metadata for the column the observation is about."}
+ ::field]
+ [:basis {:description "Where the observation came from (e.g. `:fingerprint`)."}
+ :keyword]
+ [:observation [:map {:closed true}
+ [:type {:description "Observation kind (`:single-value`, `:all-null`, `:low-cardinality`)."}
+ :keyword]
+ [:value {:description "Kind-specific detail (e.g. the distinct count for `:low-cardinality`)."}
+ :any]]]
+ [:count {:description "Number of executions in the window that surfaced this observation for this field."}
+ pos-int?]])
diff --git a/src/metabase/usage_metadata/settings.clj b/src/metabase/usage_metadata/settings.clj
new file mode 100644
index 00000000000..d7c801d5cc2
--- /dev/null
+++ b/src/metabase/usage_metadata/settings.clj
@@ -0,0 +1,39 @@
+(ns metabase.usage-metadata.settings
+ (:require
+ [metabase.settings.core :refer [defsetting]]
+ [metabase.util.i18n :as i18n]))
+
+(defsetting usage-metadata-enabled?
+ (i18n/deferred-tru "Whether usage-driven metadata batch processing is enabled.")
+ :encryption :no
+ :type :boolean
+ :default false
+ :visibility :internal
+ :export? false
+ :audit :never)
+
+(defsetting usage-metadata-retention-days
+ (i18n/deferred-tru "How many days of usage metadata rollups to retain.")
+ :encryption :no
+ :type :integer
+ :default 90
+ :visibility :internal
+ :export? false
+ :audit :never)
+
+(defsetting usage-metadata-schedule
+ (i18n/deferred-tru "Cron schedule (in UTC) for usage metadata batch processing.")
+ :encryption :no
+ :type :string
+ :default "0 0 2 * * ? *"
+ :visibility :internal
+ :export? false
+ :audit :never)
+
+(defsetting usage-metadata-last-completed-day
+ (i18n/deferred-tru "Internal watermark for the last completed usage metadata day.")
+ :encryption :no
+ :type :string
+ :visibility :internal
+ :export? false
+ :audit :never)
diff --git a/src/metabase/usage_metadata/store.clj b/src/metabase/usage_metadata/store.clj
new file mode 100644
index 00000000000..42fb560a7d1
--- /dev/null
+++ b/src/metabase/usage_metadata/store.clj
@@ -0,0 +1,72 @@
+(ns metabase.usage-metadata.store
+ (:require
+ [metabase.usage-metadata.models.source-dimension-daily]
+ [metabase.usage-metadata.models.source-dimension-profile-daily]
+ [metabase.usage-metadata.models.source-metric-daily]
+ [metabase.usage-metadata.models.source-segment-composite-daily]
+ [metabase.usage-metadata.models.source-segment-daily]
+ [toucan2.core :as t2]))
+
+(def ^:private insert-chunk-size
+ "Chunk size for bulk inserts. Keeps us under the Postgres ~65535 bind-parameter
+ cap even for wide rows."
+ 1000)
+
+(defn- chunked-insert!
+ [model rows]
+ (when (seq rows)
+ (doseq [chunk (partition-all insert-chunk-size rows)]
+ (t2/insert! model chunk))))
+
+(defn delete-day!
+ "Delete all rollup rows for `bucket-date` across the usage metadata daily tables."
+ [bucket-date]
+ (t2/delete! :model/SourceSegmentDaily :bucket_date bucket-date)
+ (t2/delete! :model/SourceSegmentCompositeDaily :bucket_date bucket-date)
+ (t2/delete! :model/SourceMetricDaily :bucket_date bucket-date)
+ (t2/delete! :model/SourceDimensionDaily :bucket_date bucket-date)
+ (t2/delete! :model/SourceDimensionProfileDaily :bucket_date bucket-date)
+ nil)
+
+(defn insert-segment-rollups!
+ "Insert daily segment rollup rows."
+ [rows]
+ (chunked-insert! :model/SourceSegmentDaily rows)
+ nil)
+
+(defn insert-composite-rollups!
+ "Insert daily composite segment (whole-:and basket) rollup rows."
+ [rows]
+ (when (seq rows)
+ (t2/insert! :model/SourceSegmentCompositeDaily rows))
+ nil)
+
+(defn insert-metric-rollups!
+ "Insert daily metric rollup rows."
+ [rows]
+ (chunked-insert! :model/SourceMetricDaily rows)
+ nil)
+
+(defn insert-dimension-rollups!
+ "Insert daily dimension rollup rows."
+ [rows]
+ (chunked-insert! :model/SourceDimensionDaily rows)
+ nil)
+
+(defn insert-dimension-profile-rollups!
+ "Insert daily dimension profile observation rows."
+ [rows]
+ (chunked-insert! :model/SourceDimensionProfileDaily rows)
+ nil)
+
+(defn replace-day!
+ "Replace all rollup rows for `bucket-date` in one transaction."
+ [bucket-date {:keys [segments composites metrics dimensions profiles]}]
+ (t2/with-transaction [_conn]
+ (delete-day! bucket-date)
+ (insert-segment-rollups! segments)
+ (insert-composite-rollups! composites)
+ (insert-metric-rollups! metrics)
+ (insert-dimension-rollups! dimensions)
+ (insert-dimension-profile-rollups! profiles))
+ nil)
diff --git a/src/metabase/usage_metadata/task/process.clj b/src/metabase/usage_metadata/task/process.clj
new file mode 100644
index 00000000000..a5e51c87118
--- /dev/null
+++ b/src/metabase/usage_metadata/task/process.clj
@@ -0,0 +1,53 @@
+(ns metabase.usage-metadata.task.process
+ (:require
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.schedule.cron :as cron]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [metabase.task.core :as task]
+ [metabase.usage-metadata.batch :as usage-metadata.batch]
+ [metabase.usage-metadata.settings :as usage-metadata.settings]
+ [metabase.util.log :as log])
+ (:import
+ (java.util TimeZone)))
+
+(set! *warn-on-reflection* true)
+
+(defn- cron-schedule
+ "Bucket dates are always UTC days; fire the cron in UTC so the schedule hour matches the UTC day we'll close out."
+ [cron-spec]
+ (cron/schedule
+ (cron/cron-schedule cron-spec)
+ (cron/in-time-zone (TimeZone/getTimeZone "UTC"))
+ (cron/with-misfire-handling-instruction-do-nothing)))
+
+(def ^:private job-key
+ (jobs/key "metabase.task.usage-metadata-process.job"))
+
+(def ^:private trigger-key
+ (triggers/key "metabase.task.usage-metadata-process.trigger"))
+
+(task/defjob ^{org.quartz.DisallowConcurrentExecution true
+ :doc "Process usage metadata rollups from query execution history."}
+ UsageMetadataProcess
+ [_]
+ (when (usage-metadata.settings/usage-metadata-enabled?)
+ (try
+ (usage-metadata.batch/run-batch!)
+ (catch Throwable e
+ (log/error e "Error processing usage metadata batch")
+ (throw e)))))
+
+(defn- job []
+ (jobs/build
+ (jobs/of-type UsageMetadataProcess)
+ (jobs/with-identity job-key)))
+
+(defn- trigger []
+ (triggers/build
+ (triggers/with-identity trigger-key)
+ (triggers/start-now)
+ (triggers/with-schedule
+ (cron-schedule (usage-metadata.settings/usage-metadata-schedule)))))
+
+(defmethod task/init! ::UsageMetadataProcess [_]
+ (task/schedule-task! (job) (trigger)))
diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.clj
index 8df396d1969..f32bf8fd1b0 100644
--- a/test/metabase/cmd/copy_test.clj
+++ b/test/metabase/cmd/copy_test.clj
@@ -57,6 +57,11 @@
:model/SearchIndexMetadata
:model/SecurityAdvisory
:model/SemanticSearchTokenTracking
+ :model/SourceDimensionDaily
+ :model/SourceDimensionProfileDaily
+ :model/SourceMetricDaily
+ :model/SourceSegmentCompositeDaily
+ :model/SourceSegmentDaily
:model/SupportAccessGrantLog
:model/TaskHistory
:model/TaskRun
diff --git a/test/metabase/lib/normalize_test.cljc b/test/metabase/lib/normalize_test.cljc
index a3a29ddf9c5..2d5b0f6bcfb 100644
--- a/test/metabase/lib/normalize_test.cljc
+++ b/test/metabase/lib/normalize_test.cljc
@@ -104,6 +104,13 @@
"aggregation" [["count" {}]]
"filters" [["=" {} ["field" {} 1] 4]]}]})))))
+(deftest ^:parallel lazy-seq-input-test
+ (testing "Seq input (e.g. LazySeqs from json/decode+kw) normalizes"
+ (is (=? [:= {:lib/uuid string?} [:field {:lib/uuid string?} 1] 4]
+ (lib/normalize '("=" {} ("field" {} 1) 4))))
+ (is (=? [:> {:lib/uuid string?} [:field {:lib/uuid string?} 1] 0]
+ (lib/normalize '(">" {} ("field" {} 1) 0))))))
+
(deftest ^:parallel normalize-from-json-test
(let [query '{:lib/type "mbql/query"
:lib/metadata nil
diff --git a/test/metabase/metrics/api_test.clj b/test/metabase/metrics/api_test.clj
index 7f1bce6e53e..55e6b2f9074 100644
--- a/test/metabase/metrics/api_test.clj
+++ b/test/metabase/metrics/api_test.clj
@@ -4,6 +4,7 @@
[clojure.test :refer :all]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.query-processor.util :as qp.util]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[toucan2.core :as t2]))
@@ -332,3 +333,74 @@
{:definition {:expression [:metric {:lib/uuid "a"} (:id metric)]
:projections []}})]
(is (= "completed" (:status response)))))))
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | QueryExecution tracking |
+;;; +----------------------------------------------------------------------------------------------------------------+
+
+(defn- rasta-metric-executions
+ "Return `QueryExecution` rows owned by the `:rasta` user, newest first.
+ Filtering by executor keeps us from picking up unrelated rows written by
+ concurrent tests."
+ []
+ (t2/select :model/QueryExecution
+ :executor_id (mt/user->id :rasta)
+ {:order-by [[:started_at :desc]]}))
+
+(deftest dataset-leaf-records-query-execution-test
+ (testing "POST /api/metric/dataset (leaf path) writes a QueryExecution row with :context :metric"
+ (mt/test-helpers-set-global-values!
+ (binding [qp.util/*execute-async?* false]
+ (mt/with-temp [:model/Card metric {:name "QE Leaf Metric"
+ :type :metric
+ :dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]
+ (let [before (count (rasta-metric-executions))]
+ (mt/user-http-request :rasta :post 202 "metric/dataset"
+ {:definition {:expression [:metric {:lib/uuid "a"} (:id metric)]}})
+ (let [rows (rasta-metric-executions)]
+ (is (= (inc before) (count rows))
+ "exactly one new QueryExecution row for rasta")
+ (is (= :metric (:context (first rows)))
+ "the new row is tagged :context :metric"))))))))
+
+(deftest dataset-arithmetic-records-one-qe-per-leaf-test
+ (testing "POST /api/metric/dataset (arithmetic) writes one QueryExecution row per leaf"
+ (mt/test-helpers-set-global-values!
+ (binding [qp.util/*execute-async?* false]
+ (mt/with-temp [:model/Card metric-a {:name "QE Arith A"
+ :type :metric
+ :dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}
+ :model/Card metric-b {:name "QE Arith B"
+ :type :metric
+ :dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]
+ (let [before (count (rasta-metric-executions))]
+ (mt/user-http-request :rasta :post 202 "metric/dataset"
+ {:definition {:expression [:+ {}
+ [:metric {:lib/uuid "a"} (:id metric-a)]
+ [:metric {:lib/uuid "b"} (:id metric-b)]]}})
+ (let [rows (rasta-metric-executions)
+ new-rows (take (- (count rows) before) rows)]
+ (is (= 2 (count new-rows))
+ "two new QueryExecution rows, one per arithmetic leaf")
+ (is (every? #(= :metric (:context %)) new-rows)
+ "both rows are tagged :context :metric"))))))))
+
+(deftest breakout-values-records-query-execution-test
+ (testing "POST /api/metric/breakout-values writes a QueryExecution row with :context :metric"
+ (mt/test-helpers-set-global-values!
+ (binding [qp.util/*execute-async?* false]
+ (mt/with-temp [:model/Card metric {:name "QE Breakout Metric"
+ :type :metric
+ :dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]
+ (mt/user-http-request :rasta :get 200 (str "metric/" (:id metric)))
+ (let [dim-uuid (-> (mt/user-http-request :rasta :get 200 (str "metric/" (:id metric)))
+ :dimensions first :id)
+ before (count (rasta-metric-executions))]
+ (mt/user-http-request :rasta :post 200 "metric/breakout-values"
+ {:definition {:expression [:metric {:lib/uuid "a"} (:id metric)]
+ :projections [{:type :metric :id (:id metric) :lib/uuid "a"
+ :projection [[:dimension {} dim-uuid]]}]}})
+ (let [rows (rasta-metric-executions)]
+ (is (= (inc before) (count rows))
+ "exactly one new QueryExecution row")
+ (is (= :metric (:context (first rows)))))))))))
diff --git a/test/metabase/usage_metadata/batch_test.clj b/test/metabase/usage_metadata/batch_test.clj
new file mode 100644
index 00000000000..2e28965e85f
--- /dev/null
+++ b/test/metabase/usage_metadata/batch_test.clj
@@ -0,0 +1,392 @@
+(ns metabase.usage-metadata.batch-test
+ (:require
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib-be.hash :as lib-be.hash]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [metabase.usage-metadata.batch :as usage-metadata.batch]
+ [metabase.usage-metadata.models.source-dimension-daily]
+ [metabase.usage-metadata.models.source-dimension-profile-daily]
+ [metabase.usage-metadata.models.source-metric-daily]
+ [metabase.usage-metadata.models.source-segment-composite-daily]
+ [metabase.usage-metadata.models.source-segment-daily]
+ [metabase.usage-metadata.settings :as usage-metadata.settings]
+ [metabase.usage-metadata.store :as usage-metadata.store]
+ [metabase.warehouse-schema.models.field]
+ [toucan2.core :as t2])
+ (:import
+ (java.nio.charset StandardCharsets)))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once (fixtures/initialize :db))
+
+(def ^:private execution-defaults
+ {:running_time 1
+ :result_rows 1
+ :native false
+ :executor_id nil
+ :card_id nil
+ :context :ad-hoc})
+
+(defn- orders-query []
+ (let [mp (lib-be/application-database-metadata-provider (mt/id))
+ orders (lib.metadata/table mp (mt/id :orders))
+ product-id (lib.metadata/field mp (mt/id :orders :product_id))
+ subtotal (lib.metadata/field mp (mt/id :orders :subtotal))
+ created-at (lib.metadata/field mp (mt/id :orders :created_at))
+ user-id (lib.metadata/field mp (mt/id :orders :user_id))]
+ (-> (lib/query mp orders)
+ (lib/filter (lib/= product-id 1))
+ (lib/aggregate (lib/sum subtotal))
+ (lib/breakout (lib/with-temporal-bucket created-at :month))
+ (lib/breakout user-id))))
+
+(defn- cross-source-query []
+ (let [mp (lib-be/application-database-metadata-provider (mt/id))
+ venues (lib.metadata/table mp (mt/id :venues))
+ categories (lib.metadata/table mp (mt/id :categories))
+ venues-cat (lib.metadata/field mp (mt/id :venues :category_id))
+ cat-id (lib.metadata/field mp (mt/id :categories :id))
+ cat-id-join (lib/with-join-alias cat-id "Cat")]
+ (-> (lib/query mp venues)
+ (lib/join (-> (lib/join-clause categories [(lib/= venues-cat cat-id-join)])
+ (lib/with-join-alias "Cat")))
+ (lib/filter (lib/= venues-cat cat-id-join))
+ (lib/breakout (lib.metadata/field mp (mt/id :venues :price))))))
+
+(defn- native-query []
+ {:database (mt/id)
+ :type :native
+ :native {:query "SELECT 1"}})
+
+(defn- insert-query! [query-hash query]
+ (t2/insert! :model/Query
+ {:query_hash query-hash
+ :query query
+ :average_execution_time 1}))
+
+(defn- insert-query-execution! [query-hash started-at]
+ (t2/insert-returning-instances! :model/QueryExecution
+ (assoc execution-defaults
+ :hash query-hash
+ :started_at started-at)))
+
+(defn- delete-query! [query-hash]
+ (t2/delete! :model/Query :query_hash query-hash))
+
+(defn- delete-day! [bucket-date]
+ (usage-metadata.store/delete-day! bucket-date))
+
+(defn- delete-query-executions-for-day! [bucket-date]
+ (t2/delete! :model/QueryExecution
+ {:where [:and
+ [:>= :started_at (t/offset-date-time bucket-date (t/local-time 0) (t/zone-offset "Z"))]
+ [:< :started_at (t/offset-date-time (t/plus bucket-date (t/days 1)) (t/local-time 0) (t/zone-offset "Z"))]]}))
+
+(deftest target-days-test
+ (is (= [(t/local-date "2026-04-13")
+ (t/local-date "2026-04-14")]
+ (usage-metadata.batch/target-days
+ {:last-completed-day (t/local-date "2026-04-12")
+ :retention-days 90
+ :yesterday (t/local-date "2026-04-14")})))
+ (is (= [(t/local-date "2026-04-13")
+ (t/local-date "2026-04-14")]
+ (usage-metadata.batch/target-days
+ {:last-completed-day nil
+ :retention-days 2
+ :yesterday (t/local-date "2026-04-14")})))
+ (is (= [(t/local-date "2026-04-13")
+ (t/local-date "2026-04-14")]
+ (usage-metadata.batch/target-days
+ {:last-completed-day (t/local-date "2026-04-01")
+ :retention-days 2
+ :yesterday (t/local-date "2026-04-14")}))))
+
+(deftest process-day!-integration-test
+ (let [bucket-date (t/local-date "2026-04-13")
+ valid-query (orders-query)
+ query-hash (lib-be.hash/query-hash valid-query)
+ execution-at (t/offset-date-time "2026-04-13T12:00Z")
+ user-id-field (mt/id :orders :user_id)
+ original-fp (t2/select-one-fn :fingerprint :model/Field :id user-id-field)]
+ (try
+ (delete-query-executions-for-day! bucket-date)
+ (delete-day! bucket-date)
+ (t2/update! :model/Field :id user-id-field {:fingerprint {:global {:distinct-count 1, :nil% 0.0}}})
+ (insert-query! query-hash valid-query)
+ (insert-query-execution! query-hash execution-at)
+ (insert-query-execution! query-hash (t/offset-date-time "2026-04-13T13:00Z"))
+ (let [result (usage-metadata.batch/process-day! bucket-date)]
+ (is (= {} (:skipped-rows result)))
+ (is (= 2 (:query-execution-rows result)))
+ (is (= 2 (:joined-rows result)))
+ (is (= 2 (:segment-tuples result)))
+ (is (= 2 (:metric-tuples result)))
+ (is (= 4 (:dimension-tuples result)))
+ (is (= 1 (:profile-observations result)))
+ (is (= 1 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))
+ (is (= 1 (t2/count :model/SourceMetricDaily :bucket_date bucket-date)))
+ (is (= 2 (t2/count :model/SourceDimensionDaily :bucket_date bucket-date)))
+ (is (= 1 (t2/count :model/SourceDimensionProfileDaily :bucket_date bucket-date)))
+ (is (= :fingerprint
+ (t2/select-one-fn :source_basis :model/SourceDimensionProfileDaily :bucket_date bucket-date)))
+ (is (= :single-value
+ (t2/select-one-fn :observation_type :model/SourceDimensionProfileDaily :bucket_date bucket-date)))
+ (is (= "2026-04-13"
+ (usage-metadata.settings/usage-metadata-last-completed-day))))
+ (finally
+ (t2/update! :model/Field :id user-id-field {:fingerprint original-fp})
+ (delete-query! query-hash)
+ (delete-query-executions-for-day! bucket-date)
+ (delete-day! bucket-date)))))
+
+(defn- composite-orders-query []
+ (let [mp (lib-be/application-database-metadata-provider (mt/id))
+ orders (lib.metadata/table mp (mt/id :orders))
+ product-id (lib.metadata/field mp (mt/id :orders :product_id))
+ subtotal (lib.metadata/field mp (mt/id :orders :subtotal))]
+ (-> (lib/query mp orders)
+ (lib/filter (lib/and (lib/= product-id 1)
+ (lib/> subtotal 0)))
+ (lib/aggregate (lib/count)))))
+
+(deftest process-day!-composite-segment-test
+ (testing "a top-level :and filter writes both atom rollup rows AND one composite rollup row"
+ (let [bucket-date (t/local-date "2026-04-13")
+ valid-query (composite-orders-query)
+ query-hash (lib-be.hash/query-hash valid-query)
+ execution-at (t/offset-date-time "2026-04-13T12:00Z")]
+ (try
+ (delete-query-executions-for-day! bucket-date)
+ (delete-day! bucket-date)
+ (insert-query! query-hash valid-query)
+ (insert-query-execution! query-hash execution-at)
+ (let [result (usage-metadata.batch/process-day! bucket-date)]
+ (is (= 2 (:segment-tuples result)))
+ (is (= 1 (:composite-tuples result))
+ "one composite fact per execution for the two-atom :and clause")
+ (is (= 2 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))
+ (is (= 1 (t2/count :model/SourceSegmentCompositeDaily :bucket_date bucket-date)))
+ (let [composite (t2/select-one :model/SourceSegmentCompositeDaily :bucket_date bucket-date)]
+ (is (= :direct (:ownership_mode composite)))
+ (is (= 2 (:atom_count composite)))
+ (is (= 1 (:count composite)))))
+ (finally
+ (delete-query! query-hash)
+ (delete-query-executions-for-day! bucket-date)
+ (delete-day! bucket-date))))))
+
+(deftest process-day!-cross-source-predicate-test
+ (let [bucket-date (t/local-date "2026-04-13")
+ valid-query (cross-source-query)
+ query-hash (lib-be.hash/query-hash valid-query)
+ execution-at (t/offset-date-time "2026-04-13T12:00Z")]
+ (try
+ (delete-query-executions-for-day! bucket-date)
+ (delete-day! bucket-date)
+ (insert-query! query-hash valid-query)
+ (insert-query-execution! query-hash execution-at)
+ (usage-metadata.batch/process-day! bucket-date)
+ (is (= #{:mixed :projected}
+ (t2/select-fn-set :ownership_mode :model/SourceSegmentDaily :bucket_date bucket-date)))
+ (is (= 3 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))
+ (finally
+ (delete-query! query-hash)
+ (delete-query-executions-for-day! bucket-date)
+ (delete-day! bucket-date)))))
+
+(deftest reprocess-day!-does-not-advance-watermark-test
+ (let [bucket-date (t/local-date "2026-04-13")
+ valid-query (orders-query)
+ query-hash (lib-be.hash/query-hash valid-query)
+ execution-at (t/offset-date-time "2026-04-13T12:00Z")]
+ (try
+ (delete-query-executions-for-day! bucket-date)
+ (delete-day! bucket-date)
+ (insert-query! query-hash valid-query)
+ (insert-query-execution! query-hash execution-at)
+ (mt/with-temporary-setting-values [usage-metadata-last-completed-day "2026-04-12"]
+ (let [result (usage-metadata.batch/reprocess-day! bucket-date)]
+ (is (false? (:watermark-advanced? result)))
+ (is (= "2026-04-12"
+ (usage-metadata.settings/usage-metadata-last-completed-day)))
+ (is (= 1 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))))
+ (finally
+ (delete-query! query-hash)
+ (delete-query-executions-for-day! bucket-date)
+ (delete-day! bucket-date)))))
+
+(deftest run-batch!-integration-test
+ (let [day-a (t/local-date "2026-04-13")
+ day-b (t/local-date "2026-04-14")
+ old-day (t/local-date "2026-04-10")
+ valid-query (orders-query)
+ valid-hash (lib-be.hash/query-hash valid-query)
+ native-hash (lib-be.hash/query-hash (native-query))
+ missing-hash (.getBytes "missing-query" StandardCharsets/UTF_8)
+ unsupported-hash (.getBytes "unsupported-query" StandardCharsets/UTF_8)]
+ (try
+ (doseq [day [day-a day-b old-day]]
+ (delete-query-executions-for-day! day)
+ (delete-day! day))
+ (insert-query! valid-hash valid-query)
+ (insert-query! native-hash (native-query))
+ (insert-query! unsupported-hash {})
+ (insert-query-execution! valid-hash (t/offset-date-time "2026-04-13T12:00Z"))
+ (insert-query-execution! valid-hash (t/offset-date-time "2026-04-14T12:00Z"))
+ (insert-query-execution! native-hash (t/offset-date-time "2026-04-14T13:00Z"))
+ (insert-query-execution! missing-hash (t/offset-date-time "2026-04-14T14:00Z"))
+ (insert-query-execution! unsupported-hash (t/offset-date-time "2026-04-14T15:00Z"))
+ (t2/insert! :model/SourceSegmentDaily
+ {:source_type :table
+ :source_id 1
+ :ownership_mode :direct
+ :field_id 1
+ :predicate "old"
+ :bucket_date old-day
+ :count 1})
+ (mt/with-temporary-setting-values [usage-metadata-last-completed-day nil
+ usage-metadata-retention-days 2]
+ (let [summary (usage-metadata.batch/run-batch!
+ {:today (t/local-date "2026-04-15")
+ :last-completed-day nil
+ :retention-days 2})]
+ (is (= :success (:status summary)))
+ (is (= 2 (:days-processed summary)))
+ (is (= 5 (:query-execution-rows summary)))
+ (is (= 4 (:joined-rows summary)))
+ (is (= 1 (get-in summary [:skipped-rows :missing-query])))
+ (is (= 1 (get-in summary [:skipped-rows :unsupported-query])))
+ (is (= 1
+ (t2/count :model/SourceSegmentDaily :bucket_date day-a)))
+ (is (= 1
+ (t2/count :model/SourceSegmentDaily :bucket_date day-b)))
+ (is (= 0
+ (t2/count :model/SourceSegmentDaily :bucket_date old-day)))
+ (is (= "2026-04-14"
+ (usage-metadata.settings/usage-metadata-last-completed-day)))))
+ (finally
+ (delete-query! valid-hash)
+ (delete-query! native-hash)
+ (delete-query! unsupported-hash)
+ (doseq [day [day-a day-b old-day]]
+ (delete-query-executions-for-day! day)
+ (delete-day! day))))))
+
+(deftest run-batch!-retention-runs-even-on-failure-test
+ (let [day-a (t/local-date "2026-04-13")
+ day-b (t/local-date "2026-04-14")
+ old-day (t/local-date "2026-04-10")
+ valid-query (orders-query)
+ query-hash (lib-be.hash/query-hash valid-query)
+ original usage-metadata.store/replace-day!]
+ (try
+ (doseq [day [day-a day-b old-day]]
+ (delete-query-executions-for-day! day)
+ (delete-day! day))
+ (insert-query! query-hash valid-query)
+ (insert-query-execution! query-hash (t/offset-date-time "2026-04-13T12:00Z"))
+ (insert-query-execution! query-hash (t/offset-date-time "2026-04-14T12:00Z"))
+ (t2/insert! :model/SourceSegmentDaily
+ {:source_type :table
+ :source_id 1
+ :ownership_mode :direct
+ :field_id 1
+ :predicate "old"
+ :bucket_date old-day
+ :count 1})
+ (mt/with-temporary-setting-values [usage-metadata-last-completed-day nil
+ usage-metadata-retention-days 2]
+ (with-redefs [usage-metadata.store/replace-day! (fn [bucket-date payload]
+ (if (= bucket-date day-b)
+ (throw (ex-info "boom" {:bucket-date bucket-date}))
+ (original bucket-date payload)))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (usage-metadata.batch/run-batch!
+ {:today (t/local-date "2026-04-15")
+ :last-completed-day nil
+ :retention-days 2})))
+ (testing "retention cleanup runs even when the processing loop fails"
+ (is (= 0 (t2/count :model/SourceSegmentDaily :bucket_date old-day))))))
+ (finally
+ (delete-query! query-hash)
+ (doseq [day [day-a day-b old-day]]
+ (delete-query-executions-for-day! day)
+ (delete-day! day))))))
+
+(deftest process-day!-skips-rows-when-normalize-query-throws-test
+ (testing "a row whose query throws inside normalize-query is counted as :normalize-error
+ and does not block the rest of the day from rolling up"
+ (let [bucket-date (t/local-date "2026-04-13")
+ good-query (orders-query)
+ good-hash (lib-be.hash/query-hash good-query)
+ bad-hash (.getBytes "bad-query" StandardCharsets/UTF_8)
+ bad-payload {:cooked :poison}
+ original lib-be/normalize-query]
+ (try
+ (delete-query-executions-for-day! bucket-date)
+ (delete-day! bucket-date)
+ (insert-query! good-hash good-query)
+ (insert-query! bad-hash bad-payload)
+ (insert-query-execution! good-hash (t/offset-date-time "2026-04-13T12:00Z"))
+ (insert-query-execution! bad-hash (t/offset-date-time "2026-04-13T13:00Z"))
+ (with-redefs [lib-be/normalize-query (fn [stored-query]
+ (if (= stored-query bad-payload)
+ (throw (ex-info "kaboom" {:q stored-query}))
+ (original stored-query)))]
+ (mt/with-temporary-setting-values [usage-metadata-last-completed-day "2026-04-12"]
+ (let [result (usage-metadata.batch/process-day! bucket-date)]
+ (testing "the bad row is recorded as :normalize-error skip"
+ (is (= 1 (get-in result [:skipped-rows :normalize-error]))))
+ (testing "the good row's rollups still landed"
+ (is (= 1 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))
+ (is (= 1 (t2/count :model/SourceMetricDaily :bucket_date bucket-date))))
+ (testing "the watermark advances despite the per-row failure"
+ (is (true? (:watermark-advanced? result)))
+ (is (= "2026-04-13"
+ (usage-metadata.settings/usage-metadata-last-completed-day)))))))
+ (finally
+ (delete-query! good-hash)
+ (delete-query! bad-hash)
+ (delete-query-executions-for-day! bucket-date)
+ (delete-day! bucket-date))))))
+
+(deftest run-batch!-stops-on-day-failure-test
+ (let [day-a (t/local-date "2026-04-13")
+ day-b (t/local-date "2026-04-14")
+ valid-query (orders-query)
+ query-hash (lib-be.hash/query-hash valid-query)
+ original usage-metadata.store/replace-day!]
+ (try
+ (doseq [day [day-a day-b]]
+ (delete-query-executions-for-day! day)
+ (delete-day! day))
+ (insert-query! query-hash valid-query)
+ (insert-query-execution! query-hash (t/offset-date-time "2026-04-13T12:00Z"))
+ (insert-query-execution! query-hash (t/offset-date-time "2026-04-14T12:00Z"))
+ (mt/with-temporary-setting-values [usage-metadata-last-completed-day nil
+ usage-metadata-retention-days 2]
+ (with-redefs [usage-metadata.store/replace-day! (fn [bucket-date payload]
+ (if (= bucket-date day-b)
+ (throw (ex-info "boom" {:bucket-date bucket-date}))
+ (original bucket-date payload)))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"
+ (usage-metadata.batch/run-batch!
+ {:today (t/local-date "2026-04-15")
+ :last-completed-day nil
+ :retention-days 2})))
+ (is (= 1 (t2/count :model/SourceSegmentDaily :bucket_date day-a)))
+ (is (= 0 (t2/count :model/SourceSegmentDaily :bucket_date day-b)))
+ (is (= "2026-04-13"
+ (usage-metadata.settings/usage-metadata-last-completed-day)))))
+ (finally
+ (delete-query! query-hash)
+ (doseq [day [day-a day-b]]
+ (delete-query-executions-for-day! day)
+ (delete-day! day))))))
diff --git a/test/metabase/usage_metadata/core_test.clj b/test/metabase/usage_metadata/core_test.clj
new file mode 100644
index 00000000000..d54962ce6ba
--- /dev/null
+++ b/test/metabase/usage_metadata/core_test.clj
@@ -0,0 +1,90 @@
+(ns metabase.usage-metadata.core-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.usage-metadata.core :as usage-metadata]
+ [metabase.usage-metadata.insights :as insights]))
+
+(def ^:private sample-segment
+ {:predicate [:= [:field 1 nil] 1]
+ :source {:type :card, :id 99, :name "Q", :display-name "Q"}
+ :fields [{:id 1, :name "f", :display-name "F"}]
+ :count 1})
+
+(def ^:private sample-metric
+ {:source {:type :table, :id 42, :name "T", :display-name "T", :db-id 1, :schema nil}
+ :aggregation {:type :count, :field nil, :temporal-field nil, :temporal-unit nil}
+ :count 1})
+
+(def ^:private sample-dimension
+ {:source {:type :table, :id 42, :name "T", :display-name "T", :db-id 1, :schema nil}
+ :dimension {:field {:id 1, :name "f", :display-name "F"}, :temporal-unit nil, :binning nil}
+ :count 1})
+
+(def ^:private sample-suggested-segment
+ {:clause [:and [:= [:field 1 nil] 1] [:> [:field 2 nil] 0]]
+ :itemset-size 2
+ :source {:type :table, :id 42, :name "T", :display-name "T", :db-id 1, :schema nil}
+ :support 3
+ :support-ratio 1.0})
+
+(def ^:private sample-profile-observation
+ {:source {:type :table, :id 42, :name "T", :display-name "T", :db-id 1, :schema nil}
+ :field {:id 1, :name "f", :display-name "F"}
+ :basis :fingerprint
+ :observation {:type :low-cardinality, :value 5}
+ :count 1})
+
+(deftest implicit-segments-delegate-to-insights-test
+ (let [captured-args (atom nil)]
+ (with-redefs [insights/implicit-segments
+ (fn [opts]
+ (reset! captured-args opts)
+ [sample-segment])]
+ (is (= [sample-segment]
+ (usage-metadata/implicit-segments {:source-type :card, :source-id 99, :limit 3})))
+ (is (= {:source-type :card, :source-id 99, :limit 3}
+ @captured-args)))))
+
+(deftest implicit-metrics-delegate-to-insights-test
+ (let [captured-args (atom nil)]
+ (with-redefs [insights/implicit-metrics
+ (fn [opts]
+ (reset! captured-args opts)
+ [sample-metric])]
+ (is (= [sample-metric]
+ (usage-metadata/implicit-metrics {:source-type :table, :source-id 42, :limit 7})))
+ (is (= {:source-type :table, :source-id 42, :limit 7}
+ @captured-args)))))
+
+(deftest implicit-dimensions-delegate-to-insights-test
+ (let [captured-args (atom nil)]
+ (with-redefs [insights/implicit-dimensions
+ (fn [opts]
+ (reset! captured-args opts)
+ [sample-dimension])]
+ (is (= [sample-dimension]
+ (usage-metadata/implicit-dimensions {:source-type :table, :source-id 42, :limit 9})))
+ (is (= {:source-type :table, :source-id 42, :limit 9}
+ @captured-args)))))
+
+(deftest suggested-segments-delegate-to-insights-test
+ (let [captured-args (atom nil)]
+ (with-redefs [insights/suggested-segments-for-owner
+ (fn [opts]
+ (reset! captured-args opts)
+ [sample-suggested-segment])]
+ (is (= [sample-suggested-segment]
+ (usage-metadata/suggested-segments {:source-type :table, :source-id 42, :limit 5})))
+ (is (= {:source-type :table, :source-id 42, :limit 5}
+ @captured-args)))))
+
+(deftest profile-observations-delegate-to-insights-test
+ (let [captured-args (atom nil)]
+ (with-redefs [insights/profile-observations
+ (fn [opts]
+ (reset! captured-args opts)
+ [sample-profile-observation])]
+ (is (= [sample-profile-observation]
+ (usage-metadata/profile-observations {:source-type :table, :source-id 42, :limit 4})))
+ (is (= {:source-type :table, :source-id 42, :limit 4}
+ @captured-args)))))
diff --git a/test/metabase/usage_metadata/extract_test.clj b/test/metabase/usage_metadata/extract_test.clj
new file mode 100644
index 00000000000..58f51360c46
--- /dev/null
+++ b/test/metabase/usage_metadata/extract_test.clj
@@ -0,0 +1,239 @@
+(ns metabase.usage-metadata.extract-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.lib.convert :as lib.convert]
+ [metabase.lib.core :as lib]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.lib.test-util :as lib.tu]
+ [metabase.usage-metadata.extract :as extract]
+ [metabase.util.json :as json]))
+
+(defn- decode-json
+ [s]
+ (json/decode+kw s))
+
+(deftest ^:parallel query-source-table-or-card-test
+ (testing "table roots are attributed to the table"
+ (is (= {:source-type :table, :source-id (meta/id :venues)}
+ (extract/query-source-table-or-card (lib.tu/venues-query)))))
+ (testing "card roots are attributed to the card"
+ (is (= {:source-type :card, :source-id 1}
+ (extract/query-source-table-or-card (lib.tu/query-with-source-card)))))
+ (testing "model roots are still attributed as cards"
+ (is (= {:source-type :card, :source-id 1}
+ (extract/query-source-table-or-card (lib.tu/query-with-source-model)))))
+ (testing "native queries are skipped"
+ (is (nil? (extract/query-source-table-or-card (lib.tu/native-query))))))
+
+(deftest ^:parallel segment-extraction-test
+ (let [query (-> (lib.tu/venues-query)
+ (lib/filter (lib/= (meta/field-metadata :venues :price) 4))
+ (lib/filter (lib/> (meta/field-metadata :venues :id) 10)))
+ rows (:segments (extract/extract-usage-facts query))]
+ (is (= 2 (count rows)))
+ (is (= #{:direct} (set (map :ownership-mode rows))))
+ (is (= #{{:source-type :table, :source-id (meta/id :venues), :field-id (meta/id :venues :price)}
+ {:source-type :table, :source-id (meta/id :venues), :field-id (meta/id :venues :id)}}
+ (set (map #(select-keys % [:source-type :source-id :field-id]) rows))))
+ (is (= #{["=" 4] [">" 10]}
+ (set (map (fn [{:keys [predicate]}]
+ (let [decoded (decode-json predicate)]
+ [(first decoded) (last decoded)]))
+ rows))))))
+
+(deftest ^:parallel compound-filter-extraction-test
+ (let [query (lib/filter (lib.tu/venues-query)
+ (lib/or (lib/= (meta/field-metadata :venues :price) 1)
+ (lib/= (meta/field-metadata :venues :price) 2)))
+ rows (:segments (extract/extract-usage-facts query))]
+ (is (= 1 (count rows)))
+ (is (= :direct (:ownership-mode (first rows))))
+ (is (= (meta/id :venues :price) (:field-id (first rows))))
+ (is (= "or"
+ (first (decode-json (:predicate (first rows))))))))
+
+(deftest ^:parallel metric-extraction-test
+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/aggregate (lib/sum (meta/field-metadata :orders :subtotal)))
+ (lib/breakout (lib/with-temporal-bucket (meta/field-metadata :orders :created-at) :month))
+ (lib/breakout (meta/field-metadata :orders :product-id)))
+ rows (:metrics (extract/extract-usage-facts query))]
+ (is (= [{:source-type :table
+ :source-id (meta/id :orders)
+ :ownership-mode :direct
+ :agg :sum
+ :agg-field-id (meta/id :orders :subtotal)
+ :temporal-field-id (meta/id :orders :created-at)
+ :temporal-unit :month}]
+ rows))))
+
+(deftest ^:parallel metric-extraction-without-temporal-breakout-test
+ (let [query (-> (lib.tu/venues-query)
+ (lib/aggregate (lib/count)))
+ rows (:metrics (extract/extract-usage-facts query))]
+ (is (= [{:source-type :table
+ :source-id (meta/id :venues)
+ :ownership-mode :direct
+ :agg :count
+ :agg-field-id nil
+ :temporal-field-id nil
+ :temporal-unit nil}]
+ rows))))
+
+(deftest ^:parallel dimension-extraction-test
+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/breakout (lib/with-temporal-bucket (meta/field-metadata :orders :created-at) :month))
+ (lib/breakout (lib/with-binning (meta/field-metadata :orders :subtotal)
+ {:strategy :num-bins, :num-bins 10})))
+ rows (:dimensions (extract/extract-usage-facts query))]
+ (is (= [{:source-type :table
+ :source-id (meta/id :orders)
+ :ownership-mode :direct
+ :field-id (meta/id :orders :created-at)
+ :temporal-unit :month
+ :binning nil}
+ {:source-type :table
+ :source-id (meta/id :orders)
+ :ownership-mode :direct
+ :field-id (meta/id :orders :subtotal)
+ :temporal-unit nil
+ :binning "{\"num-bins\":10,\"strategy\":\"num-bins\"}"}]
+ rows))
+ (is (= {:num-bins 10, :strategy "num-bins"}
+ (decode-json (:binning (second rows)))))))
+
+(deftest ^:parallel joined-field-is-owned-by-joined-source-test
+ (let [query (-> (lib.tu/query-with-join)
+ (lib/filter (lib/= (meta/field-metadata :venues :price) 1))
+ (lib/filter (lib/= (lib/with-join-alias (meta/field-metadata :categories :name) "Cat") "Pizza")))
+ rows (:segments (extract/extract-usage-facts query))]
+ (is (= #{{:source-type :table, :source-id (meta/id :venues), :ownership-mode :direct, :field-id (meta/id :venues :price)}
+ {:source-type :table, :source-id (meta/id :categories), :ownership-mode :direct, :field-id (meta/id :categories :name)}}
+ (set (map #(select-keys % [:source-type :source-id :ownership-mode :field-id]) rows))))))
+
+(deftest ^:parallel source-card-fields-stay-card-owned-test
+ (let [query (-> (lib.tu/query-with-source-model)
+ (lib/append-stage)
+ (lib/filter -1 (lib/= (meta/field-metadata :checkins :user-id) 10)))
+ rows (:segments (extract/extract-usage-facts query))]
+ (is (= 1 (count rows)))
+ (is (= {:source-type :card, :source-id 1, :ownership-mode :direct, :field-id (meta/id :checkins :user-id)}
+ (select-keys (first rows) [:source-type :source-id :ownership-mode :field-id])))))
+
+(deftest ^:parallel composite-extraction-single-owner-test
+ (testing "a top-level :and clause produces exactly one composite fact on a single-owner query"
+ (let [query (lib/filter (lib.tu/venues-query)
+ (lib/and (lib/= (meta/field-metadata :venues :price) 4)
+ (lib/> (meta/field-metadata :venues :id) 10)))
+ facts (:composites (extract/extract-usage-facts query))]
+ (is (= 1 (count facts)))
+ (let [{:keys [source-type source-id ownership-mode clause atom-fingerprints atom-count]} (first facts)]
+ (is (= :table source-type))
+ (is (= (meta/id :venues) source-id))
+ (is (= :direct ownership-mode))
+ (is (= 2 atom-count))
+ (is (= 2 (count atom-fingerprints)))
+ (is (= atom-fingerprints (vec (sort atom-fingerprints)))
+ "atom-fingerprints should be sorted for stable basket identity")
+ (is (= "and" (first (decode-json clause))))
+ (testing "atom fingerprints round-trip to canonical atom JSON"
+ (is (= #{"=" ">"} (set (map (comp first decode-json) atom-fingerprints)))))))))
+
+(deftest ^:parallel composite-extraction-single-atom-is-skipped-test
+ (testing "single-atom top-level filters emit NO composite (atom rollup already captures them)"
+ (let [query (lib/filter (lib.tu/venues-query)
+ (lib/= (meta/field-metadata :venues :price) 4))]
+ (is (empty? (:composites (extract/extract-usage-facts query)))))))
+
+(deftest ^:parallel composite-extraction-accumulated-filters-combine-into-one-basket-test
+ (testing "two separate lib/filter calls form a single composite basket (the stage's implicit :and)"
+ (let [query (-> (lib.tu/venues-query)
+ (lib/filter (lib/= (meta/field-metadata :venues :price) 4))
+ (lib/filter (lib/> (meta/field-metadata :venues :id) 10)))
+ facts (:composites (extract/extract-usage-facts query))]
+ (is (= 1 (count facts)))
+ (is (= 2 (:atom-count (first facts)))))))
+
+(deftest ^:parallel composite-extraction-multi-owner-test
+ (testing ":and spanning multiple owners yields :mixed + per-owner :projected composite facts"
+ (let [query (lib/filter (lib.tu/query-with-join)
+ (lib/and (lib/= (meta/field-metadata :venues :price) 1)
+ (lib/= (lib/with-join-alias (meta/field-metadata :categories :name) "Cat") "Pizza")))
+ facts (:composites (extract/extract-usage-facts query))]
+ (is (= 3 (count facts)))
+ (is (= 1 (count (filter #(= :mixed (:ownership-mode %)) facts))))
+ (is (= #{{:source-type :table, :source-id (meta/id :venues), :ownership-mode :projected}
+ {:source-type :table, :source-id (meta/id :categories), :ownership-mode :projected}}
+ (set (map #(select-keys % [:source-type :source-id :ownership-mode])
+ (filter #(= :projected (:ownership-mode %)) facts)))))
+ (testing "all projected rows share the same clause+atom fingerprints as the mixed row"
+ (let [fingerprints (into #{} (map (juxt :clause :atom-fingerprints)) facts)]
+ (is (= 1 (count fingerprints))))))))
+
+(deftest ^:parallel cross-source-predicate-produces-mixed-and-projected-test
+ (let [query (-> (lib.tu/query-with-join)
+ (lib/filter (lib/= (meta/field-metadata :venues :category-id)
+ (lib/with-join-alias (meta/field-metadata :categories :id) "Cat"))))
+ rows (:segments (extract/extract-usage-facts query))]
+ (is (= 3 (count rows)))
+ (is (= 1 (count (filter #(= :mixed (:ownership-mode %)) rows))))
+ (is (= #{{:source-type :table, :source-id (meta/id :venues), :ownership-mode :projected, :field-id (meta/id :venues :category-id)}
+ {:source-type :table, :source-id (meta/id :categories), :ownership-mode :projected, :field-id (meta/id :categories :id)}}
+ (set (map #(select-keys % [:source-type :source-id :ownership-mode :field-id])
+ (filter #(= :projected (:ownership-mode %)) rows)))))))
+
+(def ^:private metric-id-1 200)
+(def ^:private metric-id-2 201)
+
+(def ^:private metrics-metadata-provider
+ (lib.tu/mock-metadata-provider
+ meta/metadata-provider
+ {:cards [{:id metric-id-1
+ :name "Sum Subtotal"
+ :type :metric
+ :database-id (meta/id)
+ :table-id (meta/id :orders)
+ :dataset-query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/aggregate (lib/sum (meta/field-metadata :orders :subtotal)))
+ lib.convert/->legacy-MBQL)}
+ {:id metric-id-2
+ :name "Sum Tax"
+ :type :metric
+ :database-id (meta/id)
+ :table-id (meta/id :orders)
+ :dataset-query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/aggregate (lib/sum (meta/field-metadata :orders :tax)))
+ lib.convert/->legacy-MBQL)}]}))
+
+(deftest ^:parallel metric-extraction-only-records-primitive-aggregations-test
+ (testing "primitive Sum(field) aggregation emits a row"
+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/aggregate (lib/sum (meta/field-metadata :orders :subtotal))))
+ rows (:metrics (extract/extract-usage-facts query))]
+ (is (= 1 (count rows)))
+ (is (= :sum (:agg (first rows))))
+ (is (= (meta/id :orders :subtotal) (:agg-field-id (first rows))))))
+ (testing "top-level [:metric M1] reference emits no row (F001)"
+ (let [query (-> (lib/query metrics-metadata-provider (meta/table-metadata :orders))
+ (lib/aggregate [:metric {:lib/uuid (str (random-uuid))} metric-id-1]))
+ rows (:metrics (extract/extract-usage-facts query))]
+ (is (= [] rows))))
+ (testing "composite [:- [:metric M1] [:metric M2]] emits no row (F015)"
+ (let [query (-> (lib/query metrics-metadata-provider (meta/table-metadata :orders))
+ (lib/aggregate (lib/- [:metric {:lib/uuid (str (random-uuid))} metric-id-1]
+ [:metric {:lib/uuid (str (random-uuid))} metric-id-2])))
+ rows (:metrics (extract/extract-usage-facts query))]
+ (is (= [] rows))))
+ (testing "composite Sum(A) - Sum(B) emits no row (F015)"
+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/aggregate (lib/- (lib/sum (meta/field-metadata :orders :subtotal))
+ (lib/sum (meta/field-metadata :orders :tax)))))
+ rows (:metrics (extract/extract-usage-facts query))]
+ (is (= [] rows))))
+ (testing "Sum(coalesce(A, 0)) emits a row with field A captured"
+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/aggregate (lib/sum (lib/coalesce (meta/field-metadata :orders :subtotal) 0))))
+ rows (:metrics (extract/extract-usage-facts query))]
+ (is (= 1 (count rows)))
+ (is (= :sum (:agg (first rows))))
+ (is (= (meta/id :orders :subtotal) (:agg-field-id (first rows)))))))
diff --git a/test/metabase/usage_metadata/insights_test.clj b/test/metabase/usage_metadata/insights_test.clj
new file mode 100644
index 00000000000..34408b973dd
--- /dev/null
+++ b/test/metabase/usage_metadata/insights_test.clj
@@ -0,0 +1,248 @@
+(ns metabase.usage-metadata.insights-test
+ (:require
+ [clojure.core.memoize :as memoize]
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [metabase.usage-metadata.extract :as usage-metadata.extract]
+ [metabase.usage-metadata.insights :as insights]
+ [metabase.usage-metadata.models.source-segment-composite-daily]
+ [metabase.util.json :as json]
+ [toucan2.core :as t2]))
+
+(use-fixtures :once (fixtures/initialize :db))
+
+(def ^:private mine-itemsets @#'insights/mine-itemsets)
+(def ^:private closed-only @#'insights/closed-only)
+(def ^:private itemset-support @#'insights/itemset-support)
+(def ^:private any-atom-support @#'insights/any-atom-support)
+(def ^:private rebuild-and-clause @#'insights/rebuild-and-clause)
+(def ^:private relative-support-ok? @#'insights/relative-support-ok?)
+(def ^:private existing-composite-atomsets @#'insights/existing-composite-atomsets)
+(def ^:private composite-atomsets-memo @#'insights/existing-composite-atomsets*-memo)
+
+(deftest ^:parallel itemset-support-counts-containing-baskets-weighted-by-count-test
+ (let [baskets [{:atoms #{:a :b :c} :count 3}
+ {:atoms #{:a :b} :count 2}
+ {:atoms #{:a :c} :count 1}
+ {:atoms #{:b} :count 4}]]
+ (testing "support = sum of basket counts for baskets containing ALL atoms in the itemset"
+ (is (= 5 (itemset-support baskets #{:a :b})))
+ (is (= 4 (itemset-support baskets #{:a :c})))
+ (is (= 3 (itemset-support baskets #{:a :b :c})))
+ (is (= 0 (itemset-support baskets #{:z}))))
+ (testing "any-atom-support = sum of basket counts for baskets containing ANY atom in the itemset"
+ (is (= 10 (any-atom-support baskets #{:a :b})))
+ (is (= 6 (any-atom-support baskets #{:a}))))
+ (testing "a basket contributes to an itemset's support exactly once per its own count"
+ (doseq [basket baskets]
+ (let [contributes? (every? (:atoms basket) #{:a :b})]
+ (is (= (if contributes? (:count basket) 0)
+ (itemset-support [basket] #{:a :b}))))))))
+
+(deftest ^:parallel mine-itemsets-worked-example-test
+ (testing "design-doc worked example: baskets {a1,a2,a3}+{a1,a2,a4} yield the {a1,a2} closed itemset"
+ (let [baskets [{:atoms #{:a1 :a2 :a3} :count 1}
+ {:atoms #{:a1 :a2 :a4} :count 1}]
+ mined (mine-itemsets baskets)
+ closed (closed-only mined)]
+ (is (= {[:a1 :a2] 2} mined)
+ "absolute support floor of 2 keeps only the pair that co-occurs in both baskets")
+ (is (= {[:a1 :a2] 2} closed)
+ "closed filter has nothing to collapse when only one itemset survives"))))
+
+(deftest ^:parallel closed-only-drops-subsumed-itemsets-of-equal-support-test
+ (testing "if {a,b} and {a,b,c} have equal support, closed filter drops the smaller subset"
+ (is (= {[:a :b :c] 2}
+ (closed-only {[:a :b] 2
+ [:a :b :c] 2}))))
+ (testing "subsets with strictly greater support are preserved"
+ (is (= {[:a :b] 5
+ [:a :b :c] 2}
+ (closed-only {[:a :b] 5
+ [:a :b :c] 2}))))
+ (testing "unrelated itemsets are untouched"
+ (is (= {[:a :b] 3 [:c :d] 3}
+ (closed-only {[:a :b] 3 [:c :d] 3})))))
+
+(deftest ^:parallel mine-itemsets-respects-size-bounds-test
+ (testing "only itemsets of size ≥ fim-k-min (2) appear in output — no singletons"
+ (let [baskets [{:atoms #{:a :b} :count 5}]
+ mined (mine-itemsets baskets)]
+ (is (every? #(>= (count %) 2) (keys mined)))
+ (is (not (contains? mined [:a])))
+ (is (not (contains? mined [:b]))))))
+
+(deftest existing-segment-predicates-cached-test
+ (testing "existing-segment-predicates is TTL-memoized — repeated calls with the same opts hit the DB once"
+ (let [segment-selects (atom 0)
+ real-select t2/select
+ existing-fn @#'insights/existing-segment-predicates
+ memo-var @#'insights/existing-segment-predicates*-memo]
+ (memoize/memo-clear! memo-var)
+ (with-redefs [t2/select (fn [& args]
+ (when (and (sequential? (first args))
+ (= :model/Segment (ffirst args)))
+ (swap! segment-selects inc))
+ (apply real-select args))]
+ (existing-fn {})
+ (existing-fn {})
+ (is (= 1 @segment-selects)))
+ (memoize/memo-clear! memo-var))))
+
+(deftest existing-metric-signatures-cached-test
+ (testing "existing-metric-signatures is TTL-memoized"
+ (let [card-selects (atom 0)
+ real-select t2/select
+ existing-fn @#'insights/existing-metric-signatures
+ memo-var @#'insights/existing-metric-signatures*-memo]
+ (memoize/memo-clear! memo-var)
+ (with-redefs [t2/select (fn [& args]
+ (when (and (sequential? (first args))
+ (= :model/Card (ffirst args)))
+ (swap! card-selects inc))
+ (apply real-select args))]
+ (existing-fn)
+ (existing-fn)
+ (is (= 1 @card-selects)))
+ (memoize/memo-clear! memo-var))))
+
+;;; ---------- composite-segment read-path tests ----------
+
+(deftest ^:parallel rebuild-and-clause-test
+ (let [fp-a "[\"=\",{},[\"field\",{},1],1]"
+ fp-b "[\">\",{},[\"field\",{},2],0]"]
+ (testing "builds a properly-shaped :and MBQL clause from atom fingerprints"
+ (is (lib/clause-of-type? (rebuild-and-clause [fp-a fp-b]) :and)))
+ (testing "returns nil below fim-k-min (2) atoms"
+ (is (nil? (rebuild-and-clause [])))
+ (is (nil? (rebuild-and-clause [fp-a]))))
+ (testing "returns nil when decode-predicate drops everything below the floor"
+ (is (nil? (rebuild-and-clause [nil nil])))
+ (is (nil? (rebuild-and-clause [fp-a nil]))))))
+
+(deftest ^:parallel relative-support-ok?-test
+ (testing "ratio above 0.2 floor → true"
+ (let [baskets [{:atoms #{:a :b} :count 4}
+ {:atoms #{:a :c} :count 1}]]
+ (is (relative-support-ok? baskets [:a :b] 4))))
+ (testing "ratio below 0.2 floor → false"
+ (let [baskets [{:atoms #{:a :b} :count 1}
+ {:atoms #{:a :c} :count 4}
+ {:atoms #{:b :c} :count 5}]]
+ (is (not (relative-support-ok? baskets [:a :b] 1)))))
+ (testing "zero denominator → true (boundary)"
+ (is (relative-support-ok? [] [:z] 0))))
+
+(def ^:private composite-test-bucket-date (t/local-date "2099-01-01"))
+
+(defn- composite-orders-query []
+ (let [mp (lib-be/application-database-metadata-provider (mt/id))
+ orders (lib.metadata/table mp (mt/id :orders))
+ prod-id (lib.metadata/field mp (mt/id :orders :product_id))
+ subtotal (lib.metadata/field mp (mt/id :orders :subtotal))]
+ (-> (lib/query mp orders)
+ (lib/filter (lib/and (lib/= prod-id 1)
+ (lib/> subtotal 0))))))
+
+(defn- composite-fact-for-orders []
+ (->> (composite-orders-query)
+ usage-metadata.extract/extract-usage-facts
+ :composites
+ (filter (fn [{:keys [source-type ownership-mode]}]
+ (and (= :table source-type) (= :direct ownership-mode))))
+ first))
+
+(defn- seed-composite-row!
+ [{:keys [source-type source-id clause atom-fingerprints atom-count]} cnt]
+ (t2/insert! :model/SourceSegmentCompositeDaily
+ {:source_type source-type
+ :source_id source-id
+ :ownership_mode :direct
+ :clause clause
+ :atom_fingerprints (json/encode atom-fingerprints)
+ :atom_count atom-count
+ :bucket_date composite-test-bucket-date
+ :count cnt}))
+
+(defn- cleanup-composite-rows! []
+ (t2/delete! :model/SourceSegmentCompositeDaily :bucket_date composite-test-bucket-date))
+
+(defn- composite-opts [source-id]
+ {:source-type :table
+ :source-id source-id
+ :bucket-start composite-test-bucket-date
+ :bucket-end composite-test-bucket-date})
+
+(deftest suggested-segments-for-owner-happy-path-test
+ (cleanup-composite-rows!)
+ (memoize/memo-clear! composite-atomsets-memo)
+ (let [fact (composite-fact-for-orders)
+ opts (composite-opts (mt/id :orders))]
+ (try
+ (seed-composite-row! fact 3)
+ (let [results (insights/suggested-segments-for-owner opts)]
+ (testing "candidate is returned when no saved Segment matches"
+ (is (seq results)))
+ (testing "top candidate is a valid :and MBQL clause attributed to the right source"
+ (let [{:keys [clause itemset-size source]} (first results)]
+ (is (lib/clause-of-type? clause :and))
+ (is (= (:atom-count fact) itemset-size))
+ (is (= :table (:type source)))
+ (is (= (mt/id :orders) (:id source))))))
+ (finally
+ (cleanup-composite-rows!)
+ (memoize/memo-clear! composite-atomsets-memo)))))
+
+(deftest suggested-segments-for-owner-skips-saved-segment-match-test
+ (cleanup-composite-rows!)
+ (memoize/memo-clear! composite-atomsets-memo)
+ (let [fact (composite-fact-for-orders)
+ opts (composite-opts (mt/id :orders))]
+ (try
+ (seed-composite-row! fact 3)
+ (testing "precondition: candidate is present without a saved Segment"
+ (is (seq (insights/suggested-segments-for-owner opts))))
+ (memoize/memo-clear! composite-atomsets-memo)
+ (mt/with-temp [:model/Segment _seg {:table_id (mt/id :orders)
+ :definition (composite-orders-query)}]
+ (let [results (insights/suggested-segments-for-owner opts)]
+ (testing "candidate is filtered out when a saved Segment has the same atom-set"
+ (is (not-any? (fn [{:keys [source itemset-size]}]
+ (and (= :table (:type source))
+ (= (mt/id :orders) (:id source))
+ (= (:atom-count fact) itemset-size)))
+ results)))))
+ (finally
+ (cleanup-composite-rows!)
+ (memoize/memo-clear! composite-atomsets-memo)))))
+
+(deftest existing-composite-atomsets-cached-test
+ (testing "existing-composite-atomsets is TTL-memoized — repeated calls hit the DB once"
+ (memoize/memo-clear! composite-atomsets-memo)
+ (let [segment-selects (atom 0)
+ real-select t2/select]
+ (try
+ (with-redefs [t2/select (fn [& args]
+ (when (and (sequential? (first args))
+ (= :model/Segment (ffirst args)))
+ (swap! segment-selects inc))
+ (apply real-select args))]
+ (existing-composite-atomsets {:source-type :table :source-id (mt/id :orders)})
+ (existing-composite-atomsets {:source-type :table :source-id (mt/id :orders)})
+ (is (= 1 @segment-selects)))
+ (finally
+ (memoize/memo-clear! composite-atomsets-memo))))))
+
+(deftest suggested-segments-for-owner-empty-when-no-rows-test
+ (cleanup-composite-rows!)
+ (memoize/memo-clear! composite-atomsets-memo)
+ (try
+ (is (= [] (insights/suggested-segments-for-owner
+ (composite-opts (mt/id :orders)))))
+ (finally
+ (memoize/memo-clear! composite-atomsets-memo))))
diff --git a/test/metabase/usage_metadata/store_test.clj b/test/metabase/usage_metadata/store_test.clj
new file mode 100644
index 00000000000..197ba270d31
--- /dev/null
+++ b/test/metabase/usage_metadata/store_test.clj
@@ -0,0 +1,93 @@
+(ns metabase.usage-metadata.store-test
+ (:require
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [metabase.usage-metadata.models.source-dimension-daily]
+ [metabase.usage-metadata.models.source-dimension-profile-daily]
+ [metabase.usage-metadata.models.source-metric-daily]
+ [metabase.usage-metadata.models.source-segment-daily]
+ [metabase.usage-metadata.settings :as usage-metadata.settings]
+ [metabase.usage-metadata.store :as usage-metadata.store]
+ [toucan2.core :as t2]))
+
+(use-fixtures :once (fixtures/initialize :db))
+
+(deftest replace-day!-test
+ (let [bucket-date (t/local-date 2026 4 15)
+ payload-a {:segments [{:source_type :table, :source_id 1, :ownership_mode :direct, :field_id 10, :predicate "pred-a", :bucket_date bucket-date, :count 2}
+ {:source_type nil, :source_id nil, :ownership_mode :mixed, :field_id nil, :predicate "pred-mixed", :bucket_date bucket-date, :count 1}]
+ :metrics [{:source_type :card, :source_id 2, :ownership_mode :direct, :agg_type :count, :agg_field_id nil, :temporal_field_id nil, :temporal_unit nil, :bucket_date bucket-date, :count 3}]
+ :dimensions [{:source_type :table, :source_id 1, :ownership_mode :direct, :field_id 11, :temporal_unit :month, :binning nil, :bucket_date bucket-date, :count 4}]
+ :profiles [{:source_type :table, :source_id 1, :field_id 11, :source_basis :fingerprint, :observation_type :single-value, :observation_value nil, :bucket_date bucket-date, :count 4}]}
+ payload-b {:segments [{:source_type :table, :source_id 1, :ownership_mode :projected, :field_id 12, :predicate "pred-b", :bucket_date bucket-date, :count 5}]
+ :metrics []
+ :dimensions []
+ :profiles []}]
+ (try
+ (usage-metadata.store/delete-day! bucket-date)
+ (usage-metadata.store/replace-day! bucket-date payload-a)
+ (is (= 2 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))
+ (is (= 1 (t2/count :model/SourceMetricDaily :bucket_date bucket-date)))
+ (is (= 1 (t2/count :model/SourceDimensionDaily :bucket_date bucket-date)))
+ (is (= 1 (t2/count :model/SourceDimensionProfileDaily :bucket_date bucket-date)))
+ (usage-metadata.store/replace-day! bucket-date payload-b)
+ (is (= 1 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))
+ (is (= 0 (t2/count :model/SourceMetricDaily :bucket_date bucket-date)))
+ (is (= 0 (t2/count :model/SourceDimensionDaily :bucket_date bucket-date)))
+ (is (= 0 (t2/count :model/SourceDimensionProfileDaily :bucket_date bucket-date)))
+ (is (= {:source_type :table, :ownership_mode :projected, :field_id 12, :predicate "pred-b", :count 5}
+ (select-keys (t2/select-one :model/SourceSegmentDaily :bucket_date bucket-date)
+ [:source_type :ownership_mode :field_id :predicate :count])))
+ (finally
+ (usage-metadata.store/delete-day! bucket-date)))))
+
+(deftest replace-day!-does-not-affect-other-days-test
+ (let [day-a (t/local-date 2026 4 15)
+ day-b (t/local-date 2026 4 16)]
+ (try
+ (usage-metadata.store/delete-day! day-a)
+ (usage-metadata.store/delete-day! day-b)
+ (usage-metadata.store/replace-day! day-a {:segments [{:source_type :table, :source_id 1, :ownership_mode :direct, :field_id 10, :predicate "pred-a", :bucket_date day-a, :count 1}]
+ :metrics []
+ :dimensions []
+ :profiles []})
+ (usage-metadata.store/replace-day! day-b {:segments [{:source_type :table, :source_id 1, :ownership_mode :direct, :field_id 11, :predicate "pred-b", :bucket_date day-b, :count 1}]
+ :metrics []
+ :dimensions []
+ :profiles []})
+ (usage-metadata.store/replace-day! day-a {:segments [{:source_type :table, :source_id 1, :ownership_mode :direct, :field_id 12, :predicate "pred-c", :bucket_date day-a, :count 1}]
+ :metrics []
+ :dimensions []
+ :profiles []})
+ (is (= #{"pred-b"}
+ (t2/select-fn-set :predicate :model/SourceSegmentDaily :bucket_date day-b)))
+ (finally
+ (usage-metadata.store/delete-day! day-a)
+ (usage-metadata.store/delete-day! day-b)))))
+
+(deftest replace-day!-transaction-rolls-back-on-failure-test
+ (let [bucket-date (t/local-date 2026 4 15)
+ original {:segments [{:source_type :table, :source_id 1, :ownership_mode :direct, :field_id 10, :predicate "pred-a", :bucket_date bucket-date, :count 2}]
+ :metrics []
+ :dimensions []
+ :profiles []}]
+ (try
+ (usage-metadata.store/delete-day! bucket-date)
+ (usage-metadata.store/replace-day! bucket-date original)
+ (with-redefs [usage-metadata.store/insert-segment-rollups! (fn [_] (throw (ex-info "boom" {})))]
+ (is (thrown? clojure.lang.ExceptionInfo
+ (usage-metadata.store/replace-day! bucket-date original))))
+ (is (= #{"pred-a"}
+ (t2/select-fn-set :predicate :model/SourceSegmentDaily :bucket_date bucket-date)))
+ (finally
+ (usage-metadata.store/delete-day! bucket-date)))))
+
+(deftest usage-metadata-settings-test
+ (is (false? (usage-metadata.settings/usage-metadata-enabled?)))
+ (is (= 90 (usage-metadata.settings/usage-metadata-retention-days)))
+ (is (= "0 0 2 * * ? *" (usage-metadata.settings/usage-metadata-schedule)))
+ (mt/with-temporary-setting-values [usage-metadata-last-completed-day "2026-04-14"]
+ (is (= "2026-04-14"
+ (usage-metadata.settings/usage-metadata-last-completed-day)))))
diff --git a/test/metabase/usage_metadata/task/process_test.clj b/test/metabase/usage_metadata/task/process_test.clj
new file mode 100644
index 00000000000..85294a11f37
--- /dev/null
+++ b/test/metabase/usage_metadata/task/process_test.clj
@@ -0,0 +1,37 @@
+(ns metabase.usage-metadata.task.process-test
+ (:require
+ [clojure.test :refer :all]
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [metabase.task.core :as task]
+ [metabase.test :as mt]
+ [metabase.usage-metadata.batch]
+ [metabase.usage-metadata.task.process :as usage-metadata.task.process]))
+
+(set! *warn-on-reflection* true)
+
+(defn- scheduled-job []
+ (first (filter #(= "metabase.task.usage-metadata-process.job" (:key %))
+ (mt/scheduler-current-tasks))))
+
+(deftest init!-schedules-usage-metadata-job-test
+ (mt/with-temp-scheduler!
+ (mt/with-temporary-setting-values [usage-metadata-schedule "0 15 2 * * ? *"]
+ (task/init! ::usage-metadata.task.process/UsageMetadataProcess)
+ (let [job (scheduled-job)]
+ (is (= "metabase.task.usage-metadata-process.job" (:key job)))
+ (is (= "0 15 2 * * ? *"
+ (get-in job [:triggers 0 :cron-schedule])))))))
+
+(deftest usage-metadata-job-respects-enabled-setting-test
+ (mt/with-temp-scheduler!
+ (task/init! ::usage-metadata.task.process/UsageMetadataProcess)
+ (let [ran? (promise)]
+ (with-redefs [metabase.usage-metadata.batch/run-batch! (fn []
+ (deliver ran? true))]
+ (mt/with-temporary-setting-values [usage-metadata-enabled? false]
+ (task/trigger-now! (jobs/key "metabase.task.usage-metadata-process.job"))
+ (Thread/sleep 200)
+ (is (= ::timeout (deref ran? 50 ::timeout))))
+ (mt/with-temporary-setting-values [usage-metadata-enabled? true]
+ (task/trigger-now! (jobs/key "metabase.task.usage-metadata-process.job"))
+ (is (true? (deref ran? 5000 ::timeout))))))))