The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Analyze query executions for metadata (#72726)
John Swanson984e06d3a1bc
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 576c26ac848..3e2bd69ada0 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2417,6 +2417,23 @@:model/Field:model/Table}}+ usage-metadata+ {:team "UX West"+ :api #{metabase.usage-metadata.init}+ :model-imports #{:model/Card+ :model/Field+ :model/Query+ :model/QueryExecution+ :model/Segment+ :model/Table}+ :uses #{app-db+ lib+ lib-be+ models+ settings+ task+ util}}+user-key-value{:team "UX West":api #{metabase.user-key-value.apidiff --git a/dev/src/dev/usage_metadata.clj b/dev/src/dev/usage_metadata.cljnew file mode 100644index 00000000000..991d1791ef6--- /dev/null+++ b/dev/src/dev/usage_metadata.clj@@ -0,0 +1,64 @@+(ns dev.usage-metadata+ (:require+ [java-time.api :as t]+ [metabase.usage-metadata.batch :as usage-metadata.batch]+ [metabase.usage-metadata.core :as usage-metadata]+ [metabase.usage-metadata.models.source-dimension-daily]+ [metabase.usage-metadata.models.source-dimension-profile-daily]+ [metabase.usage-metadata.models.source-metric-daily]+ [metabase.usage-metadata.models.source-segment-daily]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(defn today-utc+ "Today's UTC date as a LocalDate."+ []+ (t/local-date (t/offset-date-time (t/zone-offset "Z"))))++(defn process-day!+ "Reprocess `bucket-date` without advancing the usage metadata watermark."+ [bucket-date]+ (usage-metadata.batch/reprocess-day! bucket-date))++(defn process-today!+ "Reprocess today's UTC bucket without advancing the usage metadata watermark."+ []+ (process-day! (today-utc)))++(defn rollups-for-day+ "Fetch usage metadata rollups for `bucket-date`, ordered by descending count."+ [bucket-date]+ {:segments (t2/select :model/SourceSegmentDaily+ :bucket_date bucket-date+ {:order-by [[:count :desc]]})+ :metrics (t2/select :model/SourceMetricDaily+ :bucket_date bucket-date+ {:order-by [[:count :desc]]})+ :dimensions (t2/select :model/SourceDimensionDaily+ :bucket_date bucket-date+ {:order-by [[:count :desc]]})+ :profiles (t2/select :model/SourceDimensionProfileDaily+ :bucket_date bucket-date+ {:order-by [[:count :desc]]})})++(defn rollup-counts-for-day+ "Return the number of rollup rows present for `bucket-date`."+ [bucket-date]+ {:segments (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)+ :metrics (t2/count :model/SourceMetricDaily :bucket_date bucket-date)+ :dimensions (t2/count :model/SourceDimensionDaily :bucket_date bucket-date)+ :profiles (t2/count :model/SourceDimensionProfileDaily :bucket_date bucket-date)})++(defn implicit-entities-for-table+ [table-id]+ (let [opts {:source-type :table :source-id table-id}]+ {:segments (usage-metadata/implicit-segments opts)+ :metrics (usage-metadata/implicit-metrics opts)+ :dimensions (usage-metadata/implicit-dimensions opts)}))++(defn implicit-entities+ []+ {:segments (usage-metadata/implicit-segments {})+ :metrics (usage-metadata/implicit-metrics {})+ :dimensions (usage-metadata/implicit-dimensions {})})diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.cljindex 61bdbd4ceed..fe16e1df3fa 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj@@ -135,6 +135,11 @@"SecurityAdvisory""SemanticSearchTokenTracking""Session"+ "SourceDimensionDaily"+ "SourceDimensionProfileDaily"+ "SourceMetricDaily"+ "SourceSegmentCompositeDaily"+ "SourceSegmentDaily""SupportAccessGrantLog""TableRemapping""TaskHistory"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.cljindex 09ffee3979e..62b117722f3 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj@@ -111,6 +111,11 @@:model/TableRemapping:model/Secret:model/Session+ :model/SourceDimensionDaily+ :model/SourceDimensionProfileDaily+ :model/SourceMetricDaily+ :model/SourceSegmentCompositeDaily+ :model/SourceSegmentDaily:model/SupportAccessGrantLog:model/TaskHistory:model/TaskRundiff --git a/resources/migrations/063/20260610_usage_metadata_composite_rollups.yaml b/resources/migrations/063/20260610_usage_metadata_composite_rollups.yamlnew file mode 100644index 00000000000..0f00024a539--- /dev/null+++ b/resources/migrations/063/20260610_usage_metadata_composite_rollups.yaml@@ -0,0 +1,80 @@+databaseChangeLog:+ - changeSet:+ id: v63.c3f8a2+ author: johnswanson+ comment: Create source_segment_composite_daily for whole-:and filter baskets+ changes:+ - createTable:+ tableName: source_segment_composite_daily+ remarks: Daily source-scoped composite segment usage rollups (whole :and baskets)+ columns:+ - column:+ name: id+ type: int+ autoIncrement: true+ remarks: Surrogate primary key+ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: source_type+ type: varchar(256)+ remarks: Owning source kind for direct or projected facts, or null for canonical mixed facts+ - column:+ name: source_id+ type: int+ remarks: Owning table or card ID for direct or projected facts, or null for canonical mixed facts+ - column:+ name: ownership_mode+ type: varchar(256)+ remarks: Ownership attribution mode such as direct, projected, or mixed+ constraints:+ nullable: false+ - column:+ name: clause+ type: ${text.type}+ remarks: Canonical serialized representation of the whole :and filter clause+ constraints:+ nullable: false+ - column:+ name: atom_fingerprints+ type: ${text.type}+ remarks: Sorted JSON array of canonical serialized atom clauses contained in the basket+ constraints:+ nullable: false+ - column:+ name: atom_count+ type: int+ remarks: Number of atomic predicates in the basket+ constraints:+ nullable: false+ - column:+ name: bucket_date+ type: date+ remarks: UTC day covered by the rollup row+ constraints:+ nullable: false+ - column:+ name: count+ type: int+ remarks: Number of observations for this composite fact on the bucket day+ constraints:+ nullable: false++ - changeSet:+ id: v63.e71b4d+ author: johnswanson+ comment: Add source_segment_composite_daily bucket/source index+ changes:+ - createIndex:+ indexName: idx_source_segment_composite_daily_bucket_source+ tableName: source_segment_composite_daily+ columns:+ - column:+ name: bucket_date+ - column:+ name: source_type+ - column:+ name: source_id+ - column:+ name: ownership_modediff --git a/resources/migrations/063/20260610_usage_metadata_rollups.yaml b/resources/migrations/063/20260610_usage_metadata_rollups.yamlnew file mode 100644index 00000000000..3a6f5e42325--- /dev/null+++ b/resources/migrations/063/20260610_usage_metadata_rollups.yaml@@ -0,0 +1,364 @@+databaseChangeLog:+ - changeSet:+ id: v63.Oong1I+ author: johnswanson+ comment: Create source_segment_daily for usage metadata rollups+ changes:+ - createTable:+ tableName: source_segment_daily+ remarks: Daily source-scoped segment usage rollups+ columns:+ - column:+ name: id+ type: int+ autoIncrement: true+ remarks: Surrogate primary key+ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: source_type+ type: varchar(254)+ remarks: Owning source kind for direct or projected facts, or null for canonical mixed facts+ - column:+ name: source_id+ type: int+ remarks: Owning table or card ID for direct or projected facts, or null for canonical mixed facts+ - column:+ name: ownership_mode+ type: varchar(254)+ remarks: Ownership attribution mode such as direct, projected, or mixed+ constraints:+ nullable: false+ - column:+ name: field_id+ type: int+ remarks: Field referenced by the segment fact when applicable+ - column:+ name: predicate+ type: ${text.type}+ remarks: Canonical serialized representation of the atomic segment predicate+ constraints:+ nullable: false+ - column:+ name: bucket_date+ type: date+ remarks: UTC day covered by the rollup row+ constraints:+ nullable: false+ - column:+ name: count+ type: int+ remarks: Number of observations for this fact on the bucket day+ constraints:+ nullable: false++ - changeSet:+ id: v63.Xoo5ue+ author: johnswanson+ comment: Create source_metric_daily for usage metadata rollups+ changes:+ - createTable:+ tableName: source_metric_daily+ remarks: Daily source-scoped metric usage rollups+ columns:+ - column:+ name: id+ type: int+ autoIncrement: true+ remarks: Surrogate primary key+ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: source_type+ type: varchar(254)+ remarks: Owning source kind for direct or projected facts, or null for canonical mixed facts+ - column:+ name: source_id+ type: int+ remarks: Owning table or card ID for direct or projected facts, or null for canonical mixed facts+ - column:+ name: ownership_mode+ type: varchar(254)+ remarks: Ownership attribution mode such as direct, projected, or mixed+ constraints:+ nullable: false+ - column:+ name: agg_type+ type: varchar(254)+ remarks: Aggregation operator for the metric fact+ constraints:+ nullable: false+ - column:+ name: agg_field_id+ type: int+ remarks: Aggregated field referenced by the metric fact when applicable+ - column:+ name: temporal_field_id+ type: int+ remarks: Breakout field supplying temporal context for the metric fact when applicable+ - column:+ name: temporal_unit+ type: varchar(254)+ remarks: Temporal bucketing unit associated with the metric fact when applicable+ - column:+ name: bucket_date+ type: date+ remarks: UTC day covered by the rollup row+ constraints:+ nullable: false+ - column:+ name: count+ type: int+ remarks: Number of observations for this fact on the bucket day+ constraints:+ nullable: false++ - changeSet:+ id: v63.Tahg3l+ author: johnswanson+ comment: Create source_dimension_daily for usage metadata rollups+ changes:+ - createTable:+ tableName: source_dimension_daily+ remarks: Daily source-scoped dimension usage rollups+ columns:+ - column:+ name: id+ type: int+ autoIncrement: true+ remarks: Surrogate primary key+ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: source_type+ type: varchar(254)+ remarks: Owning source kind for the dimension fact+ constraints:+ nullable: false+ - column:+ name: source_id+ type: int+ remarks: Owning table or card ID for the dimension fact+ constraints:+ nullable: false+ - column:+ name: ownership_mode+ type: varchar(254)+ remarks: Ownership attribution mode for the dimension fact+ constraints:+ nullable: false+ - column:+ name: field_id+ type: int+ remarks: Breakout field referenced by the dimension fact+ constraints:+ nullable: false+ - column:+ name: temporal_unit+ type: varchar(254)+ remarks: Temporal bucketing unit for temporal breakouts when applicable+ - column:+ name: binning+ type: ${text.type}+ remarks: Canonical serialized binning configuration when applicable+ - column:+ name: bucket_date+ type: date+ remarks: UTC day covered by the rollup row+ constraints:+ nullable: false+ - column:+ name: count+ type: int+ remarks: Number of observations for this fact on the bucket day+ constraints:+ nullable: false++ - changeSet:+ id: v63.ojei4O+ author: johnswanson+ comment: Create source_dimension_profile_daily for usage metadata profile observations+ changes:+ - createTable:+ tableName: source_dimension_profile_daily+ remarks: Daily source-scoped dimension profile observations+ columns:+ - column:+ name: id+ type: int+ autoIncrement: true+ remarks: Surrogate primary key+ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: source_type+ type: varchar(254)+ remarks: Owning source kind for the profiled dimension+ constraints:+ nullable: false+ - column:+ name: source_id+ type: int+ remarks: Owning table or card ID for the profiled dimension+ constraints:+ nullable: false+ - column:+ name: field_id+ type: int+ remarks: Field being profiled+ constraints:+ nullable: false+ - column:+ name: source_basis+ type: varchar(254)+ remarks: Provenance of the profile signal such as fingerprint+ constraints:+ nullable: false+ - column:+ name: observation_type+ type: varchar(254)+ remarks: Kind of profile observation such as single-value or all-null+ constraints:+ nullable: false+ - column:+ name: observation_value+ type: ${text.type}+ remarks: Optional serialized value associated with the profile observation+ - column:+ name: bucket_date+ type: date+ remarks: UTC day when the profile observation was recorded+ constraints:+ nullable: false+ - column:+ name: count+ type: int+ remarks: Number of times the observation was recorded on the bucket day+ constraints:+ nullable: false++ - changeSet:+ id: v63.Ohk8ph+ author: johnswanson+ comment: Add source_segment_daily bucket/source index+ changes:+ - createIndex:+ indexName: idx_source_segment_daily_bucket_source+ tableName: source_segment_daily+ columns:+ - column:+ name: bucket_date+ - column:+ name: source_type+ - column:+ name: source_id+ - column:+ name: ownership_mode++ - changeSet:+ id: v63.Uike3b+ author: johnswanson+ comment: Add source_segment_daily field index+ changes:+ - createIndex:+ indexName: idx_source_segment_daily_field+ tableName: source_segment_daily+ columns:+ - column:+ name: field_id++ - changeSet:+ id: v63.Chuun4+ author: johnswanson+ comment: Add source_metric_daily bucket/source index+ changes:+ - createIndex:+ indexName: idx_source_metric_daily_bucket_source+ tableName: source_metric_daily+ columns:+ - column:+ name: bucket_date+ - column:+ name: source_type+ - column:+ name: source_id+ - column:+ name: ownership_mode++ - changeSet:+ id: v63.eiQu2a+ author: johnswanson+ comment: Add source_metric_daily aggregation index+ changes:+ - createIndex:+ indexName: idx_source_metric_daily_agg+ tableName: source_metric_daily+ columns:+ - column:+ name: agg_type+ - column:+ name: agg_field_id++ - changeSet:+ id: v63.eeQu6e+ author: johnswanson+ comment: Add source_dimension_daily bucket/source index+ changes:+ - createIndex:+ indexName: idx_source_dimension_daily_bucket_source+ tableName: source_dimension_daily+ columns:+ - column:+ name: bucket_date+ - column:+ name: source_type+ - column:+ name: source_id+ - column:+ name: ownership_mode++ - changeSet:+ id: v63.mohCo0+ author: johnswanson+ comment: Add source_dimension_daily field index+ changes:+ - createIndex:+ indexName: idx_source_dimension_daily_field+ tableName: source_dimension_daily+ columns:+ - column:+ name: field_id++ - changeSet:+ id: v63.Ye2eib+ author: johnswanson+ comment: Add source_dimension_profile_daily bucket/source index+ changes:+ - createIndex:+ indexName: idx_source_dimension_profile_daily_bucket_source+ tableName: source_dimension_profile_daily+ columns:+ - column:+ name: bucket_date+ - column:+ name: source_type+ - column:+ name: source_id++ - changeSet:+ id: v63.ouZ0sh+ author: johnswanson+ comment: Add source_dimension_profile_daily field/observation index+ changes:+ - createIndex:+ indexName: idx_source_dimension_profile_daily_field+ tableName: source_dimension_profile_daily+ columns:+ - column:+ name: field_id+ - column:+ name: observation_typediff --git a/src/metabase/core/init.clj b/src/metabase/core/init.cljindex 467ea412dbb..5d7a3752330 100644--- a/src/metabase/core/init.clj+++ b/src/metabase/core/init.clj@@ -77,6 +77,7 @@[metabase.transforms.init][metabase.types.init][metabase.upload.init]+ [metabase.usage-metadata.init][metabase.user-key-value.init][metabase.users.init][metabase.version.init]diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex d5bf5c012e7..32bc406bc80 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -100,6 +100,7 @@[metabase.lib.query :as lib.query][metabase.lib.query.test-spec :as lib.query.test-spec][metabase.lib.ref :as lib.ref]+ [metabase.lib.referenced-columns :as lib.referenced-columns][metabase.lib.remove-replace :as lib.remove-replace][metabase.lib.schema :as lib.schema][metabase.lib.schema.aggregation :as lib.schema.aggregation]@@ -210,6 +211,7 @@lib.query/keep-melib.query.test-spec/keep-melib.ref/keep-me+ lib.referenced-columns/keep-melib.remove-replace/keep-melib.schema/keep-memetabase.lib.schema.util/keep-me@@ -671,6 +673,17 @@stage-number :- :int](lib.filter/filters a-query stage-number)))+(mu/defn atomic-filters :- [:maybe [:ref ::lib.schema/filters]]+ "Like [[filters]], but with any top-level `:and` clauses recursively flattened so the result is a+ list of atomic (non-`:and`) boolean filter clauses. The conjunction of the returned list is+ logically equivalent to the conjunction of [[filters]].++ **Code Health:** Healthy. This is a core API."+ ([a-query] (atomic-filters a-query -1))+ ([a-query :- ::lib.schema/query+ stage-number :- :int]+ (lib.filter/atomic-filters a-query stage-number)))+(mu/defn filterable-columns :- [:maybe [:sequential ::lib.schema.metadata/column]]"Returns column metadata for all the columns that could be used for filtering on the target stage of `a-query`.@@ -1565,6 +1578,8 @@field-ref-idfield-ref-nameref]+ [lib.referenced-columns+ referenced-columns][lib.remove-replaceremove-clauseremove-joindiff --git a/src/metabase/lib/filter.cljc b/src/metabase/lib/filter.cljcindex f5457516069..df190181396 100644--- a/src/metabase/lib/filter.cljc+++ b/src/metabase/lib/filter.cljc@@ -477,6 +477,21 @@stage-number :- [:maybe :int]](perf/not-empty (:filters (lib.util/query-stage query (clojure.core/or stage-number -1))))))+(defn- flatten-and-clause+ [a-filter]+ (if (lib.util/clause-of-type? a-filter :and)+ (into [] (mapcat flatten-and-clause) (drop 2 a-filter))+ [a-filter]))++(mu/defn atomic-filters :- [:maybe [:ref ::lib.schema/filters]]+ "Like [[filters]], but with any top-level `:and` clauses recursively flattened so the result is a+ list of atomic (non-`:and`) boolean filter clauses. The conjunction of the returned list is+ logically equivalent to the conjunction of [[filters]]."+ ([query :- ::lib.schema/query] (atomic-filters query nil))+ ([query :- ::lib.schema/query+ stage-number :- [:maybe :int]]+ (perf/not-empty (into [] (mapcat flatten-and-clause) (filters query stage-number)))))+(defn- leading-ref"Returns the first argument of `a-filter` if it is a reference clause, nil otherwise."[a-filter]diff --git a/src/metabase/lib/referenced_columns.cljc b/src/metabase/lib/referenced_columns.cljcnew file mode 100644index 00000000000..5b7d15986a0--- /dev/null+++ b/src/metabase/lib/referenced_columns.cljc@@ -0,0 +1,46 @@+(ns metabase.lib.referenced-columns+ "The natural home for `referenced-columns` would be next to `returned-columns` and friends in+ `metabase.lib.metadata.calculation`, but the implementation is built on+ [[metabase.lib.fe-util/expression-parts]], and requiring [[metabase.lib.fe-util]] from there+ would create a dependency cycle. So it lives in its own namespace downstream of `fe-util`+ instead."+ (:require+ [metabase.lib.dispatch :as lib.dispatch]+ [metabase.lib.fe-util :as lib.fe-util]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.lib.util :as lib.util]+ [metabase.util.malli :as mu]))++(defn- column-metadata?+ [x]+ (and (map? x) (= :metadata/column (:lib/type x))))++(defn- expression-parts-map?+ [x]+ (and (map? x) (= :mbql/expression-parts (:lib/type x))))++(defn- referenced-columns* [parts]+ (cond+ (column-metadata? parts) [parts]+ (expression-parts-map? parts) (into [] (mapcat referenced-columns*) (:args parts))+ :else []))++(mu/defn referenced-columns :- [:sequential ::lib.schema.metadata/column]+ "Returns the column-metadata leaves referenced by `x`, where `x` is a clause (filter, aggregation,+ expression) or a column-metadata map, resolved relative to the target stage of `query`.++ Columns appear in traversal order; duplicates are NOT removed — callers may `distinct` if+ dedup semantics are needed.++ Throws if `x` is neither an MBQL clause nor a `:metadata/column` map. Schema validation is+ off in production, so the runtime check is the gate."+ ([query x] (referenced-columns query -1 x))+ ([query :- ::lib.schema/query+ stage-number :- :int+ x]+ (when-not (or (column-metadata? x) (lib.util/clause? x))+ (throw (ex-info "referenced-columns: expected an MBQL clause or :metadata/column"+ {:dispatch-value (lib.dispatch/dispatch-value x)+ :input x})))+ (referenced-columns* (lib.fe-util/expression-parts query stage-number x))))diff --git a/src/metabase/lib/schema/info.cljc b/src/metabase/lib/schema/info.cljcindex b7341eb83f8..f3bf289166e 100644--- a/src/metabase/lib/schema/info.cljc+++ b/src/metabase/lib/schema/info.cljc@@ -19,6 +19,7 @@:cache-refresh:collection:map-tiles+ :metric:pulse:dashboard-subscription:dashboarddiff --git a/src/metabase/lib/schema/mbql_clause.cljc b/src/metabase/lib/schema/mbql_clause.cljcindex 92f1661e287..f3b5f2055a0 100644--- a/src/metabase/lib/schema/mbql_clause.cljc+++ b/src/metabase/lib/schema/mbql_clause.cljc@@ -101,9 +101,12 @@(defn- normalize-clause [x](when (sequential? x)- (if ((some-fn map? nil?) (second x))- x- (into [(first x) {:lib/uuid (str (random-uuid))}] (rest x)))))+ ;; Coerce non-vector sequentials (e.g. LazySeqs from `json/decode+kw`) to vectors so downstream+ ;; `:tuple` schemas match.+ (let [x (if (vector? x) x (vec x))]+ (if ((some-fn map? nil?) (second x))+ x+ (into [(first x) {:lib/uuid (str (random-uuid))}] (rest x))))));;; TODO: Support options more nicely - these don't allow for overriding the options, but we have a few cases where that;;; is necessary. See for example the inclusion of `string-filter-options` in [[metabase.lib.filter]].diff --git a/src/metabase/metrics/api.clj b/src/metabase/metrics/api.cljindex 8e8cfd22c58..bb4f78ea318 100644--- a/src/metabase/metrics/api.clj+++ b/src/metabase/metrics/api.clj@@ -187,6 +187,15 @@:projections (or projections []):metadata-provider provider}))+(defn- with-metric-exec-info+ "Attach `:executed-by`/`:context` to a metric-plan MBQL query so that+ [[metabase.query-processor.middleware.process-userland-query]] will write a+ `QueryExecution` row. Without this, the middleware logs+ \"Cannot save QueryExecution, missing :context\" and skips the insert."+ [query]+ (update query :info merge {:executed-by api/*current-user-id*+ :context :metric}))+(defn- execute-leaf-queries"Execute all leaf queries in parallel, collecting results eagerly.Must be called OUTSIDE streaming context to avoid JSON writer conflicts.@@ -194,7 +203,7 @@[leaves](let [uuid->future (into {}(map (fn [[uuid leaf-plan]]- [uuid (future (qp/process-query (qp/userland-query (:leaf/mbql leaf-plan))))]))+ [uuid (future (qp/process-query (qp/userland-query (with-metric-exec-info (:leaf/mbql leaf-plan)))))]))leaves)](into {}(map (fn [[uuid f]] [uuid @f]))@@ -226,7 +235,7 @@plan (lib-metric/->query-plan definition {:limit 10000})](if (= :leaf (:plan/type plan))(qp.streaming/streaming-response [rff :api]- (qp/process-query (qp/userland-query (:plan/mbql plan)) rff))+ (qp/process-query (qp/userland-query (with-metric-exec-info (:plan/mbql plan))) rff));; Arithmetic: execute leaf queries BEFORE streaming to avoid JSON writer conflicts(let [uuid->result (execute-leaf-queries (:plan/leaves plan))](qp.streaming/streaming-response [rff :api]@@ -250,7 +259,7 @@{:keys [definition]} :- ::DatasetRequest](let [definition (from-api-definition (lib-metric/metadata-provider) definition)plan (lib-metric/->query-plan definition {:limit 100 :values-only true})- result (qp/process-query (qp/userland-query (:plan/mbql plan)))+ result (qp/process-query (qp/userland-query (with-metric-exec-info (:plan/mbql plan))))col (first (get-in result [:data :cols]))values (mapv first (get-in result [:data :rows]))]{:values valuesdiff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.cljindex d905f02c4fd..00e4ed4ff63 100644--- a/src/metabase/models/resolution.clj+++ b/src/metabase/models/resolution.clj@@ -130,6 +130,11 @@:model/TransformTag metabase.transforms.models.transform-tag:model/TransformTransformTag metabase.transforms.models.transform-transform-tag:model/Undo metabase-enterprise.action-v2.models.undo+ :model/SourceDimensionDaily metabase.usage-metadata.models.source-dimension-daily+ :model/SourceDimensionProfileDaily metabase.usage-metadata.models.source-dimension-profile-daily+ :model/SourceMetricDaily metabase.usage-metadata.models.source-metric-daily+ :model/SourceSegmentCompositeDaily metabase.usage-metadata.models.source-segment-composite-daily+ :model/SourceSegmentDaily metabase.usage-metadata.models.source-segment-daily:model/User metabase.users.models.user:model/UserKeyValue metabase.user-key-value.models.user-key-value:model/UserParameterValue metabase.users.models.user-parameter-valuediff --git a/src/metabase/usage_metadata/batch.clj b/src/metabase/usage_metadata/batch.cljnew file mode 100644index 00000000000..8863db4f088--- /dev/null+++ b/src/metabase/usage_metadata/batch.clj@@ -0,0 +1,391 @@+(ns metabase.usage-metadata.batch+ (:require+ [java-time.api :as t]+ [metabase.app-db.core :as mdb]+ [metabase.lib-be.core :as lib-be]+ [metabase.usage-metadata.extract :as usage-metadata.extract]+ [metabase.usage-metadata.models.source-dimension-daily]+ [metabase.usage-metadata.models.source-dimension-profile-daily]+ [metabase.usage-metadata.models.source-metric-daily]+ [metabase.usage-metadata.models.source-segment-composite-daily]+ [metabase.usage-metadata.models.source-segment-daily]+ [metabase.usage-metadata.settings :as usage-metadata.settings]+ [metabase.usage-metadata.store :as usage-metadata.store]+ [metabase.util.json :as json]+ [metabase.util.log :as log]+ [toucan2.core :as t2]+ [toucan2.realize :as t2.realize])+ (:import+ (java.nio ByteBuffer)+ (org.postgresql.util PGobject)))++(set! *warn-on-reflection* true)++(def ^:private utc-zone-offset+ (t/zone-offset "Z"))++(def ^:private rollup-models+ [:model/SourceSegmentDaily+ :model/SourceSegmentCompositeDaily+ :model/SourceMetricDaily+ :model/SourceDimensionDaily+ :model/SourceDimensionProfileDaily])++(def ^:private low-cardinality-threshold+ 30)++(def ^:private hash-chunk-size+ "Maximum number of query hashes to put into a single `IN (?, ?, ...)` clause.+ Keeps us safely below the Postgres ~65535 bind-parameter cap when a busy+ instance accumulates a huge number of unique query hashes per day."+ 10000)++(defn- today-utc []+ (t/local-date (t/offset-date-time utc-zone-offset)))++(defn- yesterday-utc []+ (t/minus (today-utc) (t/days 1)))++(defn- parse-watermark [watermark]+ (when (seq watermark)+ (t/local-date watermark)))++(defn- hash-key [hash-bytes]+ (when hash-bytes+ (ByteBuffer/wrap ^bytes hash-bytes)))++(defn target-days+ "Return the closed UTC days the scheduled batch should process."+ ([] (target-days {:last-completed-day (parse-watermark (usage-metadata.settings/usage-metadata-last-completed-day))+ :retention-days (usage-metadata.settings/usage-metadata-retention-days)+ :yesterday (yesterday-utc)}))+ ([{:keys [last-completed-day retention-days yesterday]}]+ (let [retention-days (max 1 (or retention-days 1))+ yesterday (or yesterday (yesterday-utc))+ retention-start (t/minus yesterday (t/days (dec retention-days)))+ next-day (some-> last-completed-day (t/plus (t/days 1)))+ start-day (if next-day+ (if (neg? (compare retention-start next-day))+ next-day+ retention-start)+ retention-start)]+ (loop [day start-day+ days []]+ (if (pos? (compare day yesterday))+ days+ (recur (t/plus day (t/days 1))+ (conj days day)))))))++(defn- utc-day-start [bucket-date]+ (t/offset-date-time bucket-date (t/local-time 0) utc-zone-offset))++(defn- utc-day-end [bucket-date]+ (-> bucket-date+ utc-day-start+ (t/plus (t/days 1))))++(defn- execution-hash-counts-for-day+ "Return `[{:hash byte[], :n execution-count} ...]` for a UTC day.++ Identical hashes produce identical facts, so collapse at the DB and iterate+ unique hashes. Per-execution totals come from multiplying by `:n`."+ [bucket-date]+ (t2/select [:model/QueryExecution :hash [:%count.* :n]]+ {:where [:and+ [:>= :started_at (utc-day-start bucket-date)]+ [:< :started_at (utc-day-end bucket-date)]]+ :group-by [:hash]}))++(defn- add-skip [stats reason n]+ (update-in stats [:skipped-rows reason] (fnil + 0) n))++(defn- query->facts [stored-query]+ (try+ (let [normalized-query (lib-be/normalize-query stored-query)]+ (cond+ (empty? normalized-query)+ {:status :skip, :reason :normalize-error}++ (nil? (usage-metadata.extract/query-source-table-or-card normalized-query))+ {:status :skip, :reason :unsupported-query}++ :else+ (try+ {:status :ok+ :facts (usage-metadata.extract/extract-usage-facts normalized-query)}+ (catch Throwable _+ {:status :skip, :reason :extract-error}))))+ (catch Throwable _+ {:status :skip, :reason :normalize-error})))++(defn- aggregate-row [rows row n]+ (update rows row (fnil + 0) n))++(defn- segment-row [bucket-date {:keys [source-type source-id ownership-mode field-id predicate]}]+ {:source_type source-type+ :source_id source-id+ :ownership_mode ownership-mode+ :field_id field-id+ :predicate predicate+ :bucket_date bucket-date})++(defn- composite-row [bucket-date {:keys [source-type source-id ownership-mode clause atom-fingerprints atom-count]}]+ {:source_type source-type+ :source_id source-id+ :ownership_mode ownership-mode+ :clause clause+ :atom_fingerprints (json/encode atom-fingerprints)+ :atom_count atom-count+ :bucket_date bucket-date})++(defn- metric-row [bucket-date {:keys [source-type source-id ownership-mode agg agg-field-id temporal-field-id temporal-unit]}]+ {:source_type source-type+ :source_id source-id+ :ownership_mode ownership-mode+ :agg_type agg+ :agg_field_id agg-field-id+ :temporal_field_id temporal-field-id+ :temporal_unit temporal-unit+ :bucket_date bucket-date})++(defn- dimension-row [bucket-date {:keys [source-type source-id ownership-mode field-id temporal-unit binning]}]+ {:source_type source-type+ :source_id source-id+ :ownership_mode ownership-mode+ :field_id field-id+ :temporal_unit temporal-unit+ :binning binning+ :bucket_date bucket-date})++(defn- counts->rows [counts]+ (mapv (fn [[row count]]+ (assoc row :count count))+ counts))++(defn- accumulate-query-result [stats bucket-date n result]+ (if (= :ok (:status result))+ (let [{:keys [segments composites metrics dimensions]} (:facts result)+ add-rows (fn [acc rows row-fn]+ (reduce (fn [m fact] (aggregate-row m (row-fn bucket-date fact) n))+ acc+ rows))]+ (-> stats+ (update :segment-tuples + (* n (count segments)))+ (update :composite-tuples + (* n (count composites)))+ (update :metric-tuples + (* n (count metrics)))+ (update :dimension-tuples + (* n (count dimensions)))+ (update :segments add-rows segments segment-row)+ (update :composites add-rows composites composite-row)+ (update :metrics add-rows metrics metric-row)+ (update :dimensions add-rows dimensions dimension-row)))+ (add-skip stats (:reason result) n)))++(defn- fingerprint-observations+ [fingerprint]+ (let [distinct-count (get-in fingerprint [:global :distinct-count])+ nil-pct (get-in fingerprint [:global :nil%])]+ (cond-> []+ (= 1 distinct-count)+ (conj {:observation_type :single-value+ :observation_value nil})++ (= 1.0 nil-pct)+ (conj {:observation_type :all-null+ :observation_value nil})++ (and (integer? distinct-count)+ (> distinct-count 1)+ (<= distinct-count low-cardinality-threshold)+ (or (nil? nil-pct) (< nil-pct 1.0)))+ (conj {:observation_type :low-cardinality+ :observation_value (str distinct-count)}))))++(defn- decode-fingerprint+ [value]+ (cond+ (map? value)+ value++ (instance? PGobject value)+ (some-> ^PGobject value .getValue json/decode+kw)++ (string? value)+ (json/decode+kw value)++ :else+ nil))++(defn- field-fingerprint+ [field-id]+ (some-> (t2/select-one-fn :fingerprint :metabase_field :id field-id)+ decode-fingerprint))++(defn- profile-rows-for-dimensions+ [bucket-date dimension-rows]+ (into []+ (mapcat (fn [{:keys [source_type source_id ownership_mode field_id count]}]+ (when (and (= source_type :table)+ (= ownership_mode :direct)+ (pos-int? field_id))+ (when-let [fingerprint (field-fingerprint field_id)]+ (for [{:keys [observation_type observation_value]} (fingerprint-observations fingerprint)]+ {:source_type source_type+ :source_id source_id+ :field_id field_id+ :source_basis :fingerprint+ :observation_type observation_type+ :observation_value observation_value+ :bucket_date bucket-date+ :count count})))))+ dimension-rows))++(defn- process-query-row+ [bucket-date hash->count stats {:keys [query_hash query]}]+ (let [hash-bb (hash-key query_hash)+ n (get hash->count hash-bb)+ stats (update stats :seen-hashes conj hash-bb)]+ (if n+ (-> stats+ (update :joined-rows + n)+ (accumulate-query-result bucket-date n (query->facts query)))+ stats)))++(defn process-day!+ "Process one UTC bucket day.++ By default this advances the scheduled watermark after a successful replace-day write.+ Pass `{:advance-watermark? false}` for manual reprocessing of any day without changing+ scheduled batch state."+ ([bucket-date]+ (process-day! bucket-date {:advance-watermark? true}))+ ([bucket-date {:keys [advance-watermark?]+ :or {advance-watermark? true}}]+ (let [hash-counts (execution-hash-counts-for-day bucket-date)+ raw-hashes (mapv :hash hash-counts)+ hash->count (into {} (map (fn [{:keys [hash n]}] [(hash-key hash) n])) hash-counts)+ initial-stats {:bucket-date bucket-date+ :query-execution-rows (reduce + 0 (vals hash->count))+ :joined-rows 0+ :segment-tuples 0+ :composite-tuples 0+ :metric-tuples 0+ :dimension-tuples 0+ :profile-observations 0+ :skipped-rows {}+ :segments {}+ :composites {}+ :metrics {}+ :dimensions {}+ :seen-hashes #{}}+ after-stream (reduce+ (fn [stats hash-chunk]+ (transduce+ (map t2.realize/realize)+ (completing (partial process-query-row bucket-date hash->count))+ stats+ (mdb/streaming-reducible+ (fn [conn]+ (t2/reducible-select :conn conn [:model/Query :query_hash :query]+ :query_hash [:in hash-chunk])))))+ initial-stats+ (partition-all hash-chunk-size raw-hashes))+ seen-hashes (:seen-hashes after-stream)+ day-stats (reduce-kv+ (fn [stats hash-bb n]+ (if (contains? seen-hashes hash-bb)+ stats+ (add-skip stats :missing-query n)))+ (dissoc after-stream :seen-hashes)+ hash->count)+ dimension-rows (counts->rows (:dimensions day-stats))+ profile-rows (profile-rows-for-dimensions bucket-date dimension-rows)+ payload {:segments (counts->rows (:segments day-stats))+ :composites (counts->rows (:composites day-stats))+ :metrics (counts->rows (:metrics day-stats))+ :dimensions dimension-rows+ :profiles profile-rows}]+ (usage-metadata.store/replace-day! bucket-date payload)+ (when advance-watermark?+ (usage-metadata.settings/usage-metadata-last-completed-day! (str bucket-date)))+ (assoc day-stats+ :watermark-advanced? advance-watermark?+ :segment-rollup-rows (count (:segments payload))+ :composite-rollup-rows (count (:composites payload))+ :metric-rollup-rows (count (:metrics payload))+ :dimension-rollup-rows (count (:dimensions payload))+ :profile-observations (count (:profiles payload))))))++(defn reprocess-day!+ "Manual entrypoint for reprocessing any UTC day without advancing the scheduled watermark."+ [bucket-date]+ (process-day! bucket-date {:advance-watermark? false}))++(defn delete-expired-rollups!+ "Delete rollup rows older than the retention cutoff and return that cutoff day."+ [retention-days today]+ (let [retention-days (max 1 (or retention-days 1))+ cutoff-day (t/minus today (t/days retention-days))]+ (doseq [model rollup-models]+ (t2/delete! model :bucket_date [:< cutoff-day]))+ cutoff-day))++(defn run-batch!+ "Process all currently targetable closed UTC days and return a run summary."+ ([] (run-batch! {:today (today-utc)+ :last-completed-day (parse-watermark (usage-metadata.settings/usage-metadata-last-completed-day))+ :retention-days (usage-metadata.settings/usage-metadata-retention-days)}))+ ([{:keys [today last-completed-day retention-days]+ :or {today (today-utc)}}]+ (let [days (target-days {:last-completed-day last-completed-day+ :retention-days retention-days+ :yesterday (t/minus today (t/days 1))})+ started-ns (System/nanoTime)+ cutoff (delete-expired-rollups! retention-days today)+ initial {:status :success+ :days-targeted (count days)+ :days-processed 0+ :query-execution-rows 0+ :joined-rows 0+ :segment-tuples 0+ :metric-tuples 0+ :dimension-tuples 0+ :profile-observations 0+ :segment-rollup-rows 0+ :metric-rollup-rows 0+ :dimension-rollup-rows 0+ :skipped-rows {}+ :retention-cutoff cutoff+ :watermark-advanced-to last-completed-day}]+ (lib-be/with-metadata-provider-cache+ (let [raw-result (reduce+ (fn [summary bucket-date]+ (try+ (let [day-result (process-day! bucket-date {:advance-watermark? true})]+ (-> summary+ (update :days-processed inc)+ (update :query-execution-rows + (:query-execution-rows day-result))+ (update :joined-rows + (:joined-rows day-result))+ (update :segment-tuples + (:segment-tuples day-result))+ (update :metric-tuples + (:metric-tuples day-result))+ (update :dimension-tuples + (:dimension-tuples day-result))+ (update :profile-observations + (:profile-observations day-result))+ (update :segment-rollup-rows + (:segment-rollup-rows day-result))+ (update :metric-rollup-rows + (:metric-rollup-rows day-result))+ (update :dimension-rollup-rows + (:dimension-rollup-rows day-result))+ (update :skipped-rows #(merge-with + % (:skipped-rows day-result)))+ (assoc :watermark-advanced-to (:bucket-date day-result))))+ (catch Throwable e+ (reduced+ (assoc summary+ :status :failed+ :error e+ :watermark-advanced-to+ (parse-watermark (usage-metadata.settings/usage-metadata-last-completed-day)))))))+ initial+ days)+ elapsed-ms (long (/ (- (System/nanoTime) started-ns) 1000000))+ final-result (assoc raw-result :elapsed-ms elapsed-ms)]+ (log/info "Usage metadata batch summary" (dissoc final-result :error))+ (when-let [e (:error final-result)]+ (throw e))+ final-result)))))diff --git a/src/metabase/usage_metadata/core.clj b/src/metabase/usage_metadata/core.cljnew file mode 100644index 00000000000..e2a0192a22c--- /dev/null+++ b/src/metabase/usage_metadata/core.clj@@ -0,0 +1,41 @@+(ns metabase.usage-metadata.core+ "Public API for usage-metadata rollups."+ (:require+ [metabase.usage-metadata.insights :as insights]+ [metabase.usage-metadata.schema :as usage-metadata.schema]+ [metabase.util.malli :as mu]))++(set! *warn-on-reflection* true)++(mu/defn implicit-segments :- [:sequential ::usage-metadata.schema/implicit-segment]+ "Filter predicates users have run ad-hoc that aren't already saved as Segments — surface candidates+ for promotion to first-class Segments."+ [opts :- ::usage-metadata.schema/opts]+ (insights/implicit-segments opts))++(mu/defn implicit-metrics :- [:sequential ::usage-metadata.schema/implicit-metric]+ "Aggregation patterns users have run ad-hoc that aren't already saved as Metric cards — surface+ candidates for promotion to first-class Metrics."+ [opts :- ::usage-metadata.schema/opts]+ (insights/implicit-metrics opts))++(mu/defn implicit-dimensions :- [:sequential ::usage-metadata.schema/implicit-dimension]+ "Columns users have grouped by (breakouts) across the window — surface candidates for promotion+ to first-class dimensions or pre-aggregations."+ [opts :- ::usage-metadata.schema/opts]+ (insights/implicit-dimensions opts))++(mu/defn suggested-segments :- [:sequential ::usage-metadata.schema/suggested-segment]+ "Composite (`:and`) segment definitions that recur across a source's query history. Mined via+ Apriori FIM over composite rollup baskets: each basket is the atom-set of one stage's top-level+ `:and`. Itemsets whose atom-set matches a saved Segment's definition are filtered out, so the+ results are genuinely ad-hoc."+ [opts :- ::usage-metadata.schema/opts]+ (insights/suggested-segments-for-owner opts))++(mu/defn profile-observations :- [:sequential ::usage-metadata.schema/profile-observation]+ "Profile observations recorded for dimensions surfaced by usage-metadata — `:single-value`,+ `:all-null`, `:low-cardinality` — useful for spotting low-value columns or columns whose+ cardinality makes them suitable as facets."+ [opts :- ::usage-metadata.schema/opts]+ (insights/profile-observations opts))diff --git a/src/metabase/usage_metadata/extract.clj b/src/metabase/usage_metadata/extract.cljnew file mode 100644index 00000000000..77858d8ca1a--- /dev/null+++ b/src/metabase/usage_metadata/extract.clj@@ -0,0 +1,303 @@+(ns metabase.usage-metadata.extract+ "Extract usage facts (dimensions, segments, metrics) from a normalized MBQL query.++ Every fact is keyed on `(source-type, source-id, ownership-mode)`, where the source+ is whatever the user was actually querying — a Card or a Table. We don't drill+ through a card to its underlying table; if you query `card_42` (backed by+ `table_7`), aggregations over card columns are attributed to the card, not to+ `table_7`.++ `ownership-mode` is one of:++ - `:direct` — all field-refs in the clause share one owner; one row per+ field, attributed to that owner.+ - `:mixed` — field-refs span multiple owners (joins, cross-source+ expressions). One synthetic row with `source-type`/`source-id`+ both nil, capturing the whole-clause count.+ - `:projected` — the per-participant shadow of a `:mixed` row: for each+ distinct owner involved, emit a row attributing the+ multi-source clause to that owner. Not a card→table+ back-projection."+ (:require+ [metabase.lib.core :as lib]+ [metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.schema.util :as lib.schema.util]+ [metabase.util.json :as json]+ [metabase.util.log :as log]))++(set! *warn-on-reflection* true)++(defn- canonicalize-for-storage+ [x]+ (-> x+ lib.schema.util/remove-lib-uuids+ (lib.schema.util/sorted-maps lib.schema.common/unfussy-sorted-map)+ json/encode))++(defn query-source-table-or-card+ "Returns the source of a query (table or card ID), or `nil` if none."+ [query]+ (when (and query (not (lib/any-native-stage? query)))+ (or (when-let [table-id (lib/primary-source-table-id query)]+ {:source-type :table+ :source-id table-id})+ (when-let [card-id (lib/primary-source-card-id query)]+ {:source-type :card+ :source-id card-id}))))++(defn- owner-from-metadata+ [root-owner metadata]+ (cond+ (and (= :card (:source-type root-owner))+ (contains? #{:source/card :source/previous-stage :source/aggregations}+ (:lib/source metadata)))+ root-owner++ (pos-int? (:lib/card-id metadata))+ {:source-type :card+ :source-id (:lib/card-id metadata)}++ (pos-int? (:table-id metadata))+ {:source-type :table+ :source-id (:table-id metadata)}++ :else+ nil))++(defn- referenced-columns-safe+ [query stage-number clause]+ (try+ (lib/referenced-columns query stage-number clause)+ (catch Throwable e+ (log/debugf e "usage-metadata: referenced-columns failed (stage %s)" stage-number)+ nil)))++(defn- field-participants-of-clause+ "For one MBQL clause, returns distinct `{:field-id ... :owner ...}` maps for each referenced+ column whose `:id` resolves to a real Field and has a derivable owner."+ [query stage-number root-owner clause]+ (into []+ (comp (keep (fn [col]+ (when-let [field-id (:id col)]+ (when (pos-int? field-id)+ (when-let [owner (owner-from-metadata root-owner col)]+ {:field-id field-id :owner owner})))))+ (distinct))+ (referenced-columns-safe query stage-number clause)))++(defn- aggregation-operator-safe+ [query stage-number aggregation]+ (try+ (:operator (lib/expression-parts query stage-number aggregation))+ (catch Throwable e+ (log/debugf e "usage-metadata: aggregation operator extraction failed (stage %s)" stage-number)+ nil)))++(defn- breakout-column-safe+ [query stage-number breakout]+ (try+ (lib/breakout-column query stage-number breakout)+ (catch Throwable e+ (log/debugf e "usage-metadata: breakout-column failed (stage %s)" stage-number)+ nil)))++(defn- segment-facts-for-clause+ [query stage-number clause]+ (let [root-owner (query-source-table-or-card query)+ field-refs (field-participants-of-clause query stage-number root-owner clause)+ owners (set (map :owner field-refs))+ predicate (canonicalize-for-storage clause)]+ (cond+ (empty? field-refs)+ []++ (= 1 (count owners))+ (let [owner (first owners)]+ (mapv (fn [{:keys [field-id]}]+ (assoc owner+ :ownership-mode :direct+ :field-id field-id+ :predicate predicate))+ field-refs))++ :else+ (vec+ (concat+ [{:source-type nil+ :source-id nil+ :ownership-mode :mixed+ :field-id nil+ :predicate predicate}]+ (map (fn [{:keys [field-id owner]}]+ (assoc owner+ :ownership-mode :projected+ :field-id field-id+ :predicate predicate))+ field-refs))))))++(defn- segment-facts-for-stage+ [query stage-number]+ (into []+ (mapcat (partial segment-facts-for-clause query stage-number))+ (or (lib/filters query stage-number) [])))++;; Allowlist of primitive aggregation operators we will record. Sourced from+;; `metabase.lib.schema.aggregation` (see `::aggregation-clause-tag`).+;; Composite aggregations (`:-`, `:+`, `:case`, etc.) and references to+;; saved Metrics (`:metric`, `:measure`, `:aggregation`) are intentionally+;; excluded — for the implicit-metric discovery use case those are either+;; already explicit-by-construction or out of scope.+(def ^:private primitive-aggregation-operators+ #{:avg+ :count+ :cum-count+ :count-where+ :distinct+ :distinct-where+ :max+ :median+ :min+ :offset+ :percentile+ :share+ :stddev+ :sum+ :cum-sum+ :sum-where+ :var})++(defn- composite-facts-for-stage+ "Emit at most one composite fact per stage, treating the stage's top-level filter list as a single+ implicit-`:and` basket. `lib/atomic-filters` flattens any explicit `:and` children so the basket's+ atom membership matches the atom rollup's per-atom facts. Stages with fewer than two atoms are+ skipped (the atom rollup already captures single predicates)."+ [query stage-number]+ (let [atoms (or (lib/atomic-filters query stage-number) [])+ atom-count (count atoms)]+ (when (>= atom-count 2)+ (let [root-owner (query-source-table-or-card query)+ synthetic-and (apply lib/and atoms)+ canonical-clause (canonicalize-for-storage synthetic-and)+ canonical-atoms (vec (sort (map canonicalize-for-storage atoms)))+ field-refs (into []+ (comp (mapcat (partial field-participants-of-clause query stage-number root-owner))+ (distinct))+ atoms)+ owners (set (map :owner field-refs))+ base {:clause canonical-clause+ :atom-fingerprints canonical-atoms+ :atom-count atom-count}]+ (cond+ (empty? field-refs)+ []++ (= 1 (count owners))+ [(merge (first owners) base {:ownership-mode :direct})]++ :else+ (into [(merge base {:source-type nil :source-id nil :ownership-mode :mixed})]+ (map (fn [owner] (merge owner base {:ownership-mode :projected})))+ owners))))))++(defn- metric-bases+ [query stage-number aggregation]+ (let [root-owner (query-source-table-or-card query)+ agg-type (aggregation-operator-safe query stage-number aggregation)+ field-refs (field-participants-of-clause query stage-number root-owner aggregation)+ owners (set (map :owner field-refs))]+ (cond+ ;; Skip composite aggregations and saved-metric references — only record+ ;; primitive aggregations like sum/count/avg/etc.+ (not (contains? primitive-aggregation-operators agg-type))+ []++ (empty? field-refs)+ [{:source-type (:source-type root-owner)+ :source-id (:source-id root-owner)+ :ownership-mode :direct+ :agg agg-type+ :agg-field-id nil}]++ (= 1 (count owners))+ (mapv (fn [{:keys [field-id owner]}]+ (assoc owner+ :ownership-mode :direct+ :agg agg-type+ :agg-field-id field-id))+ field-refs)++ :else+ (vec+ (concat+ [{:source-type nil+ :source-id nil+ :ownership-mode :mixed+ :agg agg-type+ :agg-field-id nil}]+ (map (fn [{:keys [field-id owner]}]+ (assoc owner+ :ownership-mode :projected+ :agg agg-type+ :agg-field-id field-id))+ field-refs))))))++(defn- temporal-breakout-from-column+ [breakout-col]+ (when-let [unit (lib/raw-temporal-bucket breakout-col)]+ (when-let [field-id (:id breakout-col)]+ (when (pos-int? field-id)+ {:temporal-field-id field-id+ :temporal-unit unit}))))++(defn- metric-facts-for-stage+ [query stage-number]+ (let [breakout-cols (mapv (partial breakout-column-safe query stage-number)+ (or (lib/breakouts query stage-number) []))+ temporal-breakouts (into [] (keep temporal-breakout-from-column) breakout-cols)]+ (into []+ (mapcat (fn [aggregation]+ (for [base (metric-bases query stage-number aggregation)+ temporal (if (seq temporal-breakouts)+ temporal-breakouts+ [{:temporal-field-id nil+ :temporal-unit nil}])]+ (merge base temporal))))+ (or (lib/aggregations query stage-number) []))))++(defn- serialize-binning+ [binning]+ (canonicalize-for-storage+ (cond-> {:strategy (:strategy binning)}+ (:num-bins binning) (assoc :num-bins (:num-bins binning))+ (:bin-width binning) (assoc :bin-width (:bin-width binning)))))++(defn- dimension-facts-for-stage+ [query stage-number]+ (let [root-owner (query-source-table-or-card query)]+ (into []+ (keep (fn [breakout]+ (when-let [col (breakout-column-safe query stage-number breakout)]+ (when-let [field-id (:id col)]+ (when (pos-int? field-id)+ (when-let [owner (owner-from-metadata root-owner col)]+ (assoc owner+ :ownership-mode :direct+ :field-id field-id+ :temporal-unit (lib/raw-temporal-bucket col)+ :binning (some-> (lib/binning col) serialize-binning))))))))+ (or (lib/breakouts query stage-number) []))))++(defn extract-usage-facts+ "Extract fact-level usage tuples from a normalized MBQL query."+ [query]+ (reduce+ (fn [{:keys [segments composites metrics dimensions]} stage-number]+ {:segments (into segments (segment-facts-for-stage query stage-number))+ :composites (into composites (composite-facts-for-stage query stage-number))+ :metrics (into metrics (metric-facts-for-stage query stage-number))+ :dimensions (into dimensions (dimension-facts-for-stage query stage-number))})+ {:segments []+ :composites []+ :metrics []+ :dimensions []}+ (range (lib/stage-count query))))diff --git a/src/metabase/usage_metadata/init.clj b/src/metabase/usage_metadata/init.cljnew file mode 100644index 00000000000..5211ed6b3b1--- /dev/null+++ b/src/metabase/usage_metadata/init.clj@@ -0,0 +1,4 @@+(ns metabase.usage-metadata.init+ (:require+ [metabase.usage-metadata.settings]+ [metabase.usage-metadata.task.process]))diff --git a/src/metabase/usage_metadata/insights.clj b/src/metabase/usage_metadata/insights.cljnew file mode 100644index 00000000000..fda4f67ab63--- /dev/null+++ b/src/metabase/usage_metadata/insights.clj@@ -0,0 +1,579 @@+(ns metabase.usage-metadata.insights+ "Read-side helpers over usage-metadata rollups — consumer of the batch pipeline."+ (:require+ [clojure.core.memoize :as memoize]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.usage-metadata.extract :as usage-metadata.extract]+ [metabase.usage-metadata.models.source-dimension-daily]+ [metabase.usage-metadata.models.source-dimension-profile-daily]+ [metabase.usage-metadata.models.source-metric-daily]+ [metabase.usage-metadata.models.source-segment-composite-daily]+ [metabase.usage-metadata.models.source-segment-daily]+ [metabase.usage-metadata.schema :as usage-metadata.schema]+ [metabase.util.json :as json]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(defn- decode-predicate+ "Parse a canonicalized predicate JSON string back into an MBQL 5 clause."+ [predicate-json]+ (some-> predicate-json+ json/decode+kw+ lib/normalize))++(defn- decode-binning+ "Parse a canonicalized binning JSON string back into a map."+ [binning-json]+ (some-> binning-json+ json/decode+kw))++(defn- build-field-index+ "Bulk-fetch Field rows for `field-ids` and return a map of `id -> {:id :name :display-name}`.++ Avoids the per-field `Field` + `Table` + metadata-provider roundtrip that the single-row+ helper used to do."+ [field-ids]+ (let [field-ids (into #{} (filter pos-int?) field-ids)+ rows (when (seq field-ids)+ (t2/select [:model/Field :id :name :display_name] :id [:in field-ids]))]+ (into {}+ (map (fn [{:keys [id name display_name]}]+ [id {:id id+ :name name+ :display-name (or display_name name)}]))+ rows)))++(defn- build-source-index+ "Bulk-fetch Table and Card rows for the `[source-type source-id]` tuples present in+ `source-keys` and return a map keyed by `[source-type source-id]`."+ [source-keys]+ (let [by-type (group-by first source-keys)+ table-ids (into #{} (comp (keep second) (filter pos-int?)) (get by-type :table))+ card-ids (into #{} (comp (keep second) (filter pos-int?)) (get by-type :card))+ tables (when (seq table-ids)+ (t2/select [:model/Table :id :name :display_name :db_id :schema]+ :id [:in table-ids]))+ cards (when (seq card-ids)+ (t2/select [:model/Card :id :name] :id [:in card-ids]))]+ (into {}+ cat+ [(map (fn [{:keys [id name display_name db_id schema]}]+ [[:table id] {:type :table+ :id id+ :db-id db_id+ :schema schema+ :name name+ :display-name (or display_name name)}])+ tables)+ (map (fn [{:keys [id name]}]+ [[:card id] {:type :card+ :id id+ :name name+ :display-name name}])+ cards)])))++(defn- predicate-field-ids+ "Return distinct field ids referenced in a decoded predicate."+ [predicate]+ (when predicate+ (try+ (vec (lib/all-field-ids predicate))+ (catch Throwable e+ (log/debug e "usage-metadata: predicate-field-ids failed")+ []))))++;; No SQL LIMIT here: callers filter saved-signature collisions in Clojure, so a SQL LIMIT can under-deliver.+(defn- grouped-segment-rows+ "Group + sum `source_segment_daily` counts for a source filter."+ [{:keys [source-type source-id bucket-start bucket-end]}]+ (let [where (cond-> [:and+ [:in :ownership_mode ["direct" "projected"]]]+ source-type (conj [:= :source_type (name source-type)])+ source-id (conj [:= :source_id source-id])+ bucket-start (conj [:>= :bucket_date bucket-start])+ bucket-end (conj [:<= :bucket_date bucket-end]))]+ (t2/select [:model/SourceSegmentDaily+ :source_type+ :source_id+ :field_id+ :predicate+ [[:sum :count] :total_count]]+ {:where where+ :group-by [:source_type :source_id :field_id :predicate]+ :order-by [[:total_count :desc]]})))++(defn- grouped-metric-rows+ "Group + sum `source_metric_daily` counts for a source filter."+ [{:keys [source-type source-id bucket-start bucket-end]}]+ (let [where (cond-> [:and+ [:in :ownership_mode ["direct" "projected"]]]+ source-type (conj [:= :source_type (name source-type)])+ source-id (conj [:= :source_id source-id])+ bucket-start (conj [:>= :bucket_date bucket-start])+ bucket-end (conj [:<= :bucket_date bucket-end]))]+ (t2/select [:model/SourceMetricDaily+ :source_type+ :source_id+ :agg_type+ :agg_field_id+ :temporal_field_id+ :temporal_unit+ [[:sum :count] :total_count]]+ {:where where+ :group-by [:source_type :source_id :agg_type :agg_field_id :temporal_field_id :temporal_unit]+ :order-by [[:total_count :desc]]})))++(defn- grouped-dimension-rows+ "Group + sum `source_dimension_daily` counts for a source filter."+ [{:keys [source-type source-id bucket-start bucket-end]}]+ (let [where (cond-> [:and+ [:in :ownership_mode ["direct" "projected"]]]+ source-type (conj [:= :source_type (name source-type)])+ source-id (conj [:= :source_id source-id])+ bucket-start (conj [:>= :bucket_date bucket-start])+ bucket-end (conj [:<= :bucket_date bucket-end]))]+ (t2/select [:model/SourceDimensionDaily+ :source_type+ :source_id+ :field_id+ :temporal_unit+ :binning+ [[:sum :count] :total_count]]+ {:where where+ :group-by [:source_type :source_id :field_id :temporal_unit :binning]+ :order-by [[:total_count :desc]]})))++(defn- decode-atom-fingerprints [x]+ (cond+ (sequential? x) (vec x)+ (string? x) (vec (json/decode x))+ :else []))++(defn- grouped-composite-rows+ "Group + sum `source_segment_composite_daily` counts for a source filter.++ Each returned row carries the whole-clause JSON, the atom-fingerprint array, and the summed count+ across the window — the input shape expected by the FIM pass."+ [{:keys [source-type source-id bucket-start bucket-end]}]+ (let [where (cond-> [:and+ [:in :ownership_mode ["direct" "projected"]]]+ source-type (conj [:= :source_type (name source-type)])+ source-id (conj [:= :source_id source-id])+ bucket-start (conj [:>= :bucket_date bucket-start])+ bucket-end (conj [:<= :bucket_date bucket-end]))]+ (->> (t2/select [:model/SourceSegmentCompositeDaily+ :source_type+ :source_id+ :clause+ :atom_fingerprints+ :atom_count+ [[:sum :count] :total_count]]+ {:where where+ :group-by [:source_type :source_id :clause :atom_fingerprints :atom_count]+ :order-by [[:total_count :desc]]})+ (mapv (fn [row]+ (update row :atom_fingerprints decode-atom-fingerprints))))))++(defn- grouped-profile-rows+ "Group + sum `source_dimension_profile_daily` counts for a source filter."+ [{:keys [source-type source-id bucket-start bucket-end]}]+ (let [where (cond-> [:and]+ source-type (conj [:= :source_type (name source-type)])+ source-id (conj [:= :source_id source-id])+ bucket-start (conj [:>= :bucket_date bucket-start])+ bucket-end (conj [:<= :bucket_date bucket-end]))]+ (t2/select [:model/SourceDimensionProfileDaily+ :source_type+ :source_id+ :field_id+ :source_basis+ :observation_type+ :observation_value+ [[:sum :count] :total_count]]+ {:where where+ :group-by [:source_type :source_id :field_id :source_basis :observation_type :observation_value]+ :order-by [[:total_count :desc]]})))++(defn- wrap-query+ "Wrap a raw MBQL map in a full lib query using the app DB metadata-provider. Returns nil on failure."+ [database-id query-map]+ (when (and (pos-int? database-id) (seq query-map))+ (try+ (lib/query (lib-be/application-database-metadata-provider database-id) query-map)+ (catch Throwable e+ (log/debug e "Failed to wrap query for usage-metadata insights")+ nil))))++(defn- extract-facts+ [database-id query-map]+ (when-let [q (wrap-query database-id query-map)]+ (try+ (usage-metadata.extract/extract-usage-facts q)+ (catch Throwable e+ (log/debug e "Failed to extract usage facts for usage-metadata insights")+ nil))))++(def ^:private cache-ttl-ms+ (* 60 1000))++(defn- existing-segment-predicates*+ [[source-type source-id]]+ (let [where (cond-> [:and [:= :archived false]]+ (and (= source-type :table) source-id) (conj [:= :table_id source-id]))+ segments (t2/select [:model/Segment :id :table_id :definition] {:where where})+ table-ids (into #{} (comp (keep :table_id) (filter pos-int?)) segments)+ table->db (when (seq table-ids)+ (into {}+ (map (juxt :id :db_id))+ (t2/select [:model/Table :id :db_id] :id [:in table-ids])))]+ (lib-be/with-metadata-provider-cache+ (into #{}+ (mapcat (fn [{:keys [table_id definition]}]+ (when (and (pos-int? table_id) (seq definition))+ (when-let [db-id (get table->db table_id)]+ (let [facts (:segments (extract-facts db-id definition))]+ (for [{:keys [predicate]} facts+ :when predicate]+ [:table table_id predicate]))))))+ segments))))++(def ^:private existing-segment-predicates*-memo+ (memoize/ttl existing-segment-predicates* :ttl/threshold cache-ttl-ms))++(defn- existing-segment-predicates+ "Set of `[source-type source-id predicate-json]` tuples for non-archived Segments whose+ atomic filter clauses would collide with stored implicit segment predicates."+ [{:keys [source-type source-id]}]+ (existing-segment-predicates*-memo [source-type source-id]))++(defn- existing-metric-signatures*+ []+ (let [cards (t2/select [:model/Card :id :database_id :dataset_query :card_schema]+ :type "metric"+ :archived false)]+ (lib-be/with-metadata-provider-cache+ (into #{}+ (mapcat (fn [{:keys [database_id dataset_query]}]+ (when (seq dataset_query)+ (let [facts (:metrics (extract-facts database_id dataset_query))]+ (for [{:keys [source-type source-id ownership-mode+ agg agg-field-id temporal-field-id temporal-unit]} facts+ ;; only :direct signatures are comparable to stored rollup rows for this source+ :when (and source-type (= ownership-mode :direct))]+ [source-type source-id agg agg-field-id temporal-field-id temporal-unit])))))+ cards))))++(def ^:private existing-metric-signatures*-memo+ (memoize/ttl (fn [_] (existing-metric-signatures*)) :ttl/threshold cache-ttl-ms))++(defn- existing-metric-signatures+ "Set of `[source-type source-id agg-type agg-field-id temporal-field-id temporal-unit]`+ tuples for non-archived Metric cards (Cards of type `metric`). Scan is independent of opts;+ TTL-memoized with a singleton key."+ []+ (existing-metric-signatures*-memo ::all))++(mu/defn implicit-segments :- [:sequential ::usage-metadata.schema/implicit-segment]+ "Top implicit segments recorded across usage-metadata rollups.++ Predicates that correspond to *existing* saved Segments are filtered out — only truly+ ad-hoc filter patterns are returned."+ ([] (implicit-segments {}))+ ([{:keys [limit] :or {limit 10} :as opts} :- ::usage-metadata.schema/opts]+ (let [existing (existing-segment-predicates opts)+ raw-rows (remove (fn [{:keys [source_type source_id predicate]}]+ (contains? existing [source_type source_id predicate]))+ (grouped-segment-rows opts))+ ;; Decode each predicate + its field-ids in one pass so we can bulk-fetch below.+ enriched (mapv (fn [{:keys [predicate] :as row}]+ (let [decoded (decode-predicate predicate)]+ (assoc row+ ::decoded decoded+ ::field-ids (predicate-field-ids decoded))))+ raw-rows)+ source-idx (build-source-index+ (into #{} (map (juxt :source_type :source_id)) enriched))+ field-idx (build-field-index+ (into #{} (mapcat ::field-ids) enriched))]+ (into []+ (comp+ (keep (fn [row]+ (when-let [source (source-idx [(:source_type row) (:source_id row)])]+ (let [fields (into [] (keep field-idx) (::field-ids row))]+ (when (seq fields)+ {:predicate (::decoded row)+ :source source+ :fields fields+ :count (long (:total_count row))})))))+ (take limit))+ enriched))))++(mu/defn implicit-metrics :- [:sequential ::usage-metadata.schema/implicit-metric]+ "Top implicit metrics recorded across usage-metadata rollups.++ Aggregations that correspond to *existing* saved Metrics (Cards of type `metric`) are+ filtered out — only truly ad-hoc aggregation patterns are returned."+ ([] (implicit-metrics {}))+ ([{:keys [limit] :or {limit 10} :as opts} :- ::usage-metadata.schema/opts]+ (let [existing (existing-metric-signatures)+ rows (remove (fn [{:keys [source_type source_id agg_type agg_field_id temporal_field_id temporal_unit]}]+ (contains? existing [source_type source_id agg_type agg_field_id temporal_field_id temporal_unit]))+ (grouped-metric-rows opts))+ source-idx (build-source-index+ (into #{} (map (juxt :source_type :source_id)) rows))+ field-idx (build-field-index+ (into #{} (mapcat (juxt :agg_field_id :temporal_field_id)) rows))]+ (into []+ (comp+ (keep (fn [{:keys [source_type source_id agg_type agg_field_id temporal_field_id temporal_unit total_count]}]+ (when-let [source (source-idx [source_type source_id])]+ {:source source+ :aggregation {:type agg_type+ :field (field-idx agg_field_id)+ :temporal-field (field-idx temporal_field_id)+ :temporal-unit temporal_unit}+ :count (long total_count)})))+ (take limit))+ rows))))++(mu/defn implicit-dimensions :- [:sequential ::usage-metadata.schema/implicit-dimension]+ "Top implicit dimensions recorded across usage-metadata rollups."+ ([] (implicit-dimensions {}))+ ([{:keys [limit] :or {limit 10} :as opts} :- ::usage-metadata.schema/opts]+ (let [rows (grouped-dimension-rows opts)+ source-idx (build-source-index+ (into #{} (map (juxt :source_type :source_id)) rows))+ field-idx (build-field-index+ (into #{} (keep :field_id) rows))]+ (into []+ (comp+ (keep (fn [{:keys [source_type source_id field_id temporal_unit binning total_count]}]+ (when-let [source (source-idx [source_type source_id])]+ (when-let [field (field-idx field_id)]+ {:source source+ :dimension {:field field+ :temporal-unit temporal_unit+ :binning (decode-binning binning)}+ :count (long total_count)}))))+ (take limit))+ rows))))++(def ^:private fim-absolute-support-floor 2)+(def ^:private fim-relative-support-floor 0.2)+(def ^:private fim-k-min 2)+(def ^:private fim-k-max 5)+(def ^:private fim-default-limit 20)++(defn- rows->baskets+ "Project rollup rows to `{:atoms #{...} :count n}` baskets for mining.++ Coerces `total_count` to `long` so downstream FIM math (and the `:support` field) stays+ integer — MariaDB/MySQL return `SUM(int_col)` as `BigDecimal`."+ [rows]+ (into []+ (keep (fn [{:keys [atom_fingerprints total_count]}]+ (when (>= (count atom_fingerprints) fim-k-min)+ {:atoms (set atom_fingerprints)+ :count (long total_count)})))+ rows))++(defn- itemset-support+ [baskets itemset]+ (reduce (fn [acc {:keys [atoms count]}]+ (if (every? atoms itemset)+ (+ acc count)+ acc))+ 0+ baskets))++(defn- any-atom-support+ "Number of baskets containing at least one atom of `itemset` (denominator for the relative support check)."+ [baskets itemset]+ (reduce (fn [acc {:keys [atoms count]}]+ (if (some atoms itemset)+ (+ acc count)+ acc))+ 0+ baskets))++(defn- frequent-singletons+ [baskets absolute-floor]+ (let [counts (reduce (fn [m {:keys [atoms count]}]+ (reduce (fn [m a] (update m a (fnil + 0) count)) m atoms))+ {}+ baskets)]+ (into {}+ (filter (fn [[_ n]] (>= n absolute-floor)))+ counts)))++(defn- apriori-join+ "Generate k+1 candidate itemsets by joining k-itemsets sharing a k-1 prefix."+ [lk-vecs]+ (let [by-prefix (group-by (fn [is] (subvec is 0 (dec (count is)))) lk-vecs)]+ (into #{}+ (mapcat (fn [group]+ (for [a group+ b group+ :when (neg? (compare (peek a) (peek b)))]+ (conj a (peek b)))))+ (vals by-prefix))))++(defn- has-all-k-subsets?+ [lk-set candidate]+ (let [n (count candidate)]+ (every? (fn [i]+ (let [sub (into (subvec candidate 0 i) (subvec candidate (inc i)))]+ (contains? lk-set sub)))+ (range n))))++(defn- mine-itemsets+ "Apriori up to size `fim-k-max`. Returns `{itemset-vec support}` for itemsets of size ≥ `fim-k-min`."+ [baskets]+ (let [singletons (frequent-singletons baskets fim-absolute-support-floor)+ l1-vecs (vec (sort (map vector (keys singletons))))]+ (loop [lk l1-vecs+ k 1+ acc {}]+ (if (or (empty? lk) (>= k fim-k-max))+ acc+ (let [lk-set (set lk)+ candidates (apriori-join lk)+ pruned (into [] (filter (partial has-all-k-subsets? lk-set)) candidates)+ counted (into {}+ (keep (fn [c]+ (let [s (itemset-support baskets (set c))]+ (when (>= s fim-absolute-support-floor) [c s]))))+ pruned)+ next-k (inc k)+ acc (if (>= next-k fim-k-min)+ (merge acc counted)+ acc)]+ (recur (vec (sort (keys counted))) next-k acc))))))++(defn- closed-only+ "Keep only itemsets that have no proper superset of equal support — the closed frequent itemsets."+ [itemset->support]+ (let [entries (vec itemset->support)]+ (into {}+ (remove (fn [[is sup]]+ (let [is-set (set is)+ is-n (count is)]+ (some (fn [[other other-sup]]+ (and (= sup other-sup)+ (> (count other) is-n)+ (every? (set other) is-set)))+ entries))))+ entries)))++(defn- relative-support-ok?+ [baskets itemset support]+ (let [denom (any-atom-support baskets itemset)]+ (or (zero? denom)+ (>= (/ support (double denom)) fim-relative-support-floor))))++(defn- rebuild-and-clause+ [fingerprints]+ (let [atoms (into []+ (keep decode-predicate)+ fingerprints)]+ (when (>= (count atoms) fim-k-min)+ (lib/simplify-compound-filter (apply lib/and atoms)))))++(defn- existing-composite-atomsets*+ [[source-type source-id]]+ (let [where (cond-> [:and [:= :archived false]]+ (and (= source-type :table) source-id) (conj [:= :table_id source-id]))+ segments (t2/select [:model/Segment :id :table_id :definition] {:where where})+ table-ids (into #{} (comp (keep :table_id) (filter pos-int?)) segments)+ table->db (when (seq table-ids)+ (into {}+ (map (juxt :id :db_id))+ (t2/select [:model/Table :id :db_id] :id [:in table-ids])))]+ (lib-be/with-metadata-provider-cache+ (into #{}+ (mapcat (fn [{:keys [table_id definition]}]+ (when (and (pos-int? table_id) (seq definition))+ (when-let [db-id (get table->db table_id)]+ (let [facts (:composites (extract-facts db-id definition))]+ (for [{:keys [atom-fingerprints]} facts+ :when (>= (count atom-fingerprints) fim-k-min)]+ [:table table_id (set atom-fingerprints)]))))))+ segments))))++(def ^:private existing-composite-atomsets*-memo+ (memoize/ttl existing-composite-atomsets* :ttl/threshold cache-ttl-ms))++(defn- existing-composite-atomsets+ "Set of `[source-type source-id #{atom-fingerprint ...}]` tuples for non-archived Segments whose+ definitions are whole-:and baskets. Used to filter out suggestions that already exist as saved Segments."+ [{:keys [source-type source-id]}]+ (existing-composite-atomsets*-memo [source-type source-id]))++(mu/defn suggested-segments-for-owner :- [:sequential ::usage-metadata.schema/suggested-segment]+ "Suggest composite (`:and`) segment definitions that recur across a source's query history but+ have not been saved as Segments yet. Implemented as Apriori FIM over composite rollup baskets:+ each rollup row is a basket whose items are the atomic predicates of one stage's top-level `:and`.+ We mine closed frequent itemsets and reconstruct each surviving itemset as an `:and` MBQL clause.++ `:itemset-size` is bounded by `fim-k-min`/`fim-k-max` (2..5). `:support` is the weighted count of+ baskets containing ALL of the itemset's atoms (basket weight = the rollup row's `:count`).+ `:support-ratio` is `support / any-atom-support` and is floored by `fim-relative-support-floor`.++ Results are sorted by `:support` desc, then by `:itemset-size` desc — at equal support, larger+ recurring `:and`s rank higher, since they encode more user intent. Truncated to `:limit`."+ ([] (suggested-segments-for-owner {}))+ ([{:keys [limit] :or {limit fim-default-limit} :as opts} :- ::usage-metadata.schema/opts]+ (let [rows (grouped-composite-rows opts)+ by-source (group-by (juxt :source_type :source_id) rows)+ source-idx (build-source-index (keys by-source))+ candidates (into []+ (mapcat (fn [[[source-type source-id] source-rows]]+ (when-let [source (source-idx [source-type source-id])]+ (let [baskets (rows->baskets source-rows)+ existing (existing-composite-atomsets {:source-type source-type+ :source-id source-id})+ mined (closed-only (mine-itemsets baskets))]+ (for [[itemset-vec support] mined+ :let [itemset (set itemset-vec)]+ :when (and (relative-support-ok? baskets itemset-vec support)+ (not (contains? existing [source-type source-id itemset])))+ :let [clause (rebuild-and-clause itemset-vec)+ denom (any-atom-support baskets itemset-vec)]+ :when clause]+ {:clause clause+ :itemset-size (count itemset-vec)+ :source source+ :support support+ :support-ratio (if (pos? denom) (/ support (double denom)) 0.0)})))))+ by-source)]+ (into []+ (take limit)+ (sort-by (juxt (comp - :support) (comp - :itemset-size)) candidates)))))++(mu/defn profile-observations :- [:sequential ::usage-metadata.schema/profile-observation]+ "Top dimension profile observations recorded across usage-metadata rollups."+ ([] (profile-observations {}))+ ([{:keys [limit] :or {limit 10} :as opts} :- ::usage-metadata.schema/opts]+ (let [rows (grouped-profile-rows opts)+ source-idx (build-source-index+ (into #{} (map (juxt :source_type :source_id)) rows))+ field-idx (build-field-index+ (into #{} (keep :field_id) rows))]+ (into []+ (comp+ (keep (fn [{:keys [source_type source_id field_id source_basis+ observation_type observation_value total_count]}]+ (when-let [source (source-idx [source_type source_id])]+ (when-let [field (field-idx field_id)]+ {:source source+ :field field+ :basis source_basis+ :observation {:type observation_type+ :value observation_value}+ :count (long total_count)}))))+ (take limit))+ rows))))diff --git a/src/metabase/usage_metadata/models/source_dimension_daily.clj b/src/metabase/usage_metadata/models/source_dimension_daily.cljnew file mode 100644index 00000000000..24eb1f1d1b4--- /dev/null+++ b/src/metabase/usage_metadata/models/source_dimension_daily.clj@@ -0,0 +1,14 @@+(ns metabase.usage-metadata.models.source-dimension-daily+ (:require+ [metabase.models.interface :as mi]+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(methodical/defmethod t2/table-name :model/SourceDimensionDaily [_model] :source_dimension_daily)++(t2/deftransforms :model/SourceDimensionDaily+ {:source_type mi/transform-keyword+ :ownership_mode mi/transform-keyword+ :temporal_unit mi/transform-keyword})++(derive :model/SourceDimensionDaily :metabase/model)diff --git a/src/metabase/usage_metadata/models/source_dimension_profile_daily.clj b/src/metabase/usage_metadata/models/source_dimension_profile_daily.cljnew file mode 100644index 00000000000..99b295da627--- /dev/null+++ b/src/metabase/usage_metadata/models/source_dimension_profile_daily.clj@@ -0,0 +1,19 @@+(ns metabase.usage-metadata.models.source-dimension-profile-daily+ "Daily rollup of dimension-profile observations (`:single-value`, `:all-null`,+ `:low-cardinality`) for fields that showed up as direct table dimensions in+ the day's activity. Each observation is read from the field's existing+ `Field.fingerprint` (populated by the sync fingerprinter); `source_basis` is+ `:fingerprint`."+ (:require+ [metabase.models.interface :as mi]+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(methodical/defmethod t2/table-name :model/SourceDimensionProfileDaily [_model] :source_dimension_profile_daily)++(t2/deftransforms :model/SourceDimensionProfileDaily+ {:source_type mi/transform-keyword+ :source_basis mi/transform-keyword+ :observation_type mi/transform-keyword})++(derive :model/SourceDimensionProfileDaily :metabase/model)diff --git a/src/metabase/usage_metadata/models/source_metric_daily.clj b/src/metabase/usage_metadata/models/source_metric_daily.cljnew file mode 100644index 00000000000..c9af4da537e--- /dev/null+++ b/src/metabase/usage_metadata/models/source_metric_daily.clj@@ -0,0 +1,15 @@+(ns metabase.usage-metadata.models.source-metric-daily+ (:require+ [metabase.models.interface :as mi]+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(methodical/defmethod t2/table-name :model/SourceMetricDaily [_model] :source_metric_daily)++(t2/deftransforms :model/SourceMetricDaily+ {:source_type mi/transform-keyword+ :ownership_mode mi/transform-keyword+ :agg_type mi/transform-keyword+ :temporal_unit mi/transform-keyword})++(derive :model/SourceMetricDaily :metabase/model)diff --git a/src/metabase/usage_metadata/models/source_segment_composite_daily.clj b/src/metabase/usage_metadata/models/source_segment_composite_daily.cljnew file mode 100644index 00000000000..0a64cd378cb--- /dev/null+++ b/src/metabase/usage_metadata/models/source_segment_composite_daily.clj@@ -0,0 +1,13 @@+(ns metabase.usage-metadata.models.source-segment-composite-daily+ (:require+ [metabase.models.interface :as mi]+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(methodical/defmethod t2/table-name :model/SourceSegmentCompositeDaily [_model] :source_segment_composite_daily)++(t2/deftransforms :model/SourceSegmentCompositeDaily+ {:source_type mi/transform-keyword+ :ownership_mode mi/transform-keyword})++(derive :model/SourceSegmentCompositeDaily :metabase/model)diff --git a/src/metabase/usage_metadata/models/source_segment_daily.clj b/src/metabase/usage_metadata/models/source_segment_daily.cljnew file mode 100644index 00000000000..e386ae3044d--- /dev/null+++ b/src/metabase/usage_metadata/models/source_segment_daily.clj@@ -0,0 +1,13 @@+(ns metabase.usage-metadata.models.source-segment-daily+ (:require+ [metabase.models.interface :as mi]+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(methodical/defmethod t2/table-name :model/SourceSegmentDaily [_model] :source_segment_daily)++(t2/deftransforms :model/SourceSegmentDaily+ {:source_type mi/transform-keyword+ :ownership_mode mi/transform-keyword})++(derive :model/SourceSegmentDaily :metabase/model)diff --git a/src/metabase/usage_metadata/schema.clj b/src/metabase/usage_metadata/schema.cljnew file mode 100644index 00000000000..569e4042c6f--- /dev/null+++ b/src/metabase/usage_metadata/schema.clj@@ -0,0 +1,112 @@+(ns metabase.usage-metadata.schema+ "Shared malli schemas for usage-metadata public API inputs and results.++ These are the shape contracts pinned at the `metabase.usage-metadata.core` boundary and+ enforced inside `metabase.usage-metadata.insights` producers."+ (:require+ [metabase.util.malli.registry :as mr]))++(mr/def ::source-type [:enum :table :card])++(mr/def ::opts+ [:map {:closed true}+ [:source-type {:optional true, :description "Restrict to a specific source kind."}+ [:maybe ::source-type]]+ [:source-id {:optional true, :description "Restrict to a specific source id."}+ [:maybe pos-int?]]+ [:bucket-start {:optional true, :description "Inclusive lower bound on rollup bucket_date."}+ [:maybe :time/local-date]]+ [:bucket-end {:optional true, :description "Inclusive upper bound on rollup bucket_date."}+ [:maybe :time/local-date]]+ [:limit {:optional true, :description "Maximum number of results to return."}+ [:maybe pos-int?]]])++(mr/def ::source+ [:map+ [:type ::source-type]+ [:id pos-int?]+ [:name [:maybe :string]]+ [:display-name [:maybe :string]]+ ;; :db-id / :schema are present only on :table sources.+ [:db-id {:optional true} [:maybe pos-int?]]+ [:schema {:optional true} [:maybe :string]]])++(mr/def ::field+ [:map+ [:id pos-int?]+ [:name [:maybe :string]]+ [:display-name [:maybe :string]]])++(mr/def ::mbql-clause+ [:fn {:error/message "expected an MBQL clause"}+ (fn [x] (and (vector? x) (keyword? (first x))))])++(mr/def ::implicit-segment+ [:map {:closed true}+ [:predicate {:description "The MBQL filter clause as it appeared in queries, decoded from canonical storage."}+ [:maybe ::mbql-clause]]+ [:source {:description "The table or card the predicate was applied to."}+ ::source]+ [:fields {:description "Field metadata for every field referenced inside the predicate. Rows with no resolvable fields are dropped upstream."}+ [:sequential {:min 1} ::field]]+ [:count {:description "Number of query executions in the window whose stage used this exact predicate."}+ pos-int?]])++(mr/def ::implicit-metric+ [:map {:closed true}+ [:source {:description "The table or card the aggregation was applied to."}+ ::source]+ [:aggregation [:map {:closed true}+ [:type {:description "Primitive op (`:sum`, `:count`, `:avg`, …). Composite aggregations and saved-metric refs are excluded upstream."}+ :keyword]+ [:field {:description "The aggregated column's metadata (nil for `:count`)."}+ [:maybe ::field]]+ [:temporal-field {:description "The temporal breakout column joined to this aggregation, if any. nil means an untimed aggregation."}+ [:maybe ::field]]+ [:temporal-unit {:description "Bucket of the temporal breakout (`:day`, `:month`, …)."}+ [:maybe :keyword]]]]+ [:count {:description "Number of executions whose stage used this aggregation shape."}+ pos-int?]])++(mr/def ::implicit-dimension+ [:map {:closed true}+ [:source {:description "The table or card the breakout was applied to."}+ ::source]+ [:dimension [:map {:closed true}+ [:field {:description "The broken-out column's metadata."}+ ::field]+ [:temporal-unit {:description "Temporal bucket (`:day`, `:month`, …) if the breakout was temporally bucketed; nil otherwise."}+ [:maybe :keyword]]+ [:binning {:description "Decoded binning spec (`:num-bins`, `:bin-width`, `:strategy`) if the breakout was binned; nil otherwise."}+ [:maybe :map]]]]+ [:count {:description "Number of executions whose stage broke out by this exact shape."}+ pos-int?]])++(mr/def ::suggested-segment+ [:map {:closed true}+ [:clause {:description "Reconstructed `[:and ...]` MBQL clause built from the itemset's atoms — what a caller would offer the user to save as a new Segment."}+ ::mbql-clause]+ [:itemset-size {:description "Number of atomic predicates the suggestion combines (`k` in FIM terms; 2..5 bounded by `fim-k-min`/`fim-k-max`). NOT the size of the source baskets the itemset was mined from — those can be larger."}+ pos-int?]+ [:source {:description "The table or card the suggestion is attributed to. Suggestions live within a single source; cross-source composites are not mined."}+ ::source]+ [:support {:description "Weighted count of baskets containing ALL the itemset's atoms (basket weight is the rollup row's execution count). Primary ranking key."}+ pos-int?]+ [:support-ratio {:description "Fraction of baskets-touching-any-atom that also contain the full itemset. Guards against an individually popular atom dragging a co-occurrence that doesn't actually travel together. Floored by `fim-relative-support-floor`."}+ number?]])++(mr/def ::profile-observation+ [:map {:closed true}+ [:source {:description "The table or card the field lives on."}+ ::source]+ [:field {:description "Field metadata for the column the observation is about."}+ ::field]+ [:basis {:description "Where the observation came from (e.g. `:fingerprint`)."}+ :keyword]+ [:observation [:map {:closed true}+ [:type {:description "Observation kind (`:single-value`, `:all-null`, `:low-cardinality`)."}+ :keyword]+ [:value {:description "Kind-specific detail (e.g. the distinct count for `:low-cardinality`)."}+ :any]]]+ [:count {:description "Number of executions in the window that surfaced this observation for this field."}+ pos-int?]])diff --git a/src/metabase/usage_metadata/settings.clj b/src/metabase/usage_metadata/settings.cljnew file mode 100644index 00000000000..d7c801d5cc2--- /dev/null+++ b/src/metabase/usage_metadata/settings.clj@@ -0,0 +1,39 @@+(ns metabase.usage-metadata.settings+ (:require+ [metabase.settings.core :refer [defsetting]]+ [metabase.util.i18n :as i18n]))++(defsetting usage-metadata-enabled?+ (i18n/deferred-tru "Whether usage-driven metadata batch processing is enabled.")+ :encryption :no+ :type :boolean+ :default false+ :visibility :internal+ :export? false+ :audit :never)++(defsetting usage-metadata-retention-days+ (i18n/deferred-tru "How many days of usage metadata rollups to retain.")+ :encryption :no+ :type :integer+ :default 90+ :visibility :internal+ :export? false+ :audit :never)++(defsetting usage-metadata-schedule+ (i18n/deferred-tru "Cron schedule (in UTC) for usage metadata batch processing.")+ :encryption :no+ :type :string+ :default "0 0 2 * * ? *"+ :visibility :internal+ :export? false+ :audit :never)++(defsetting usage-metadata-last-completed-day+ (i18n/deferred-tru "Internal watermark for the last completed usage metadata day.")+ :encryption :no+ :type :string+ :visibility :internal+ :export? false+ :audit :never)diff --git a/src/metabase/usage_metadata/store.clj b/src/metabase/usage_metadata/store.cljnew file mode 100644index 00000000000..42fb560a7d1--- /dev/null+++ b/src/metabase/usage_metadata/store.clj@@ -0,0 +1,72 @@+(ns metabase.usage-metadata.store+ (:require+ [metabase.usage-metadata.models.source-dimension-daily]+ [metabase.usage-metadata.models.source-dimension-profile-daily]+ [metabase.usage-metadata.models.source-metric-daily]+ [metabase.usage-metadata.models.source-segment-composite-daily]+ [metabase.usage-metadata.models.source-segment-daily]+ [toucan2.core :as t2]))++(def ^:private insert-chunk-size+ "Chunk size for bulk inserts. Keeps us under the Postgres ~65535 bind-parameter+ cap even for wide rows."+ 1000)++(defn- chunked-insert!+ [model rows]+ (when (seq rows)+ (doseq [chunk (partition-all insert-chunk-size rows)]+ (t2/insert! model chunk))))++(defn delete-day!+ "Delete all rollup rows for `bucket-date` across the usage metadata daily tables."+ [bucket-date]+ (t2/delete! :model/SourceSegmentDaily :bucket_date bucket-date)+ (t2/delete! :model/SourceSegmentCompositeDaily :bucket_date bucket-date)+ (t2/delete! :model/SourceMetricDaily :bucket_date bucket-date)+ (t2/delete! :model/SourceDimensionDaily :bucket_date bucket-date)+ (t2/delete! :model/SourceDimensionProfileDaily :bucket_date bucket-date)+ nil)++(defn insert-segment-rollups!+ "Insert daily segment rollup rows."+ [rows]+ (chunked-insert! :model/SourceSegmentDaily rows)+ nil)++(defn insert-composite-rollups!+ "Insert daily composite segment (whole-:and basket) rollup rows."+ [rows]+ (when (seq rows)+ (t2/insert! :model/SourceSegmentCompositeDaily rows))+ nil)++(defn insert-metric-rollups!+ "Insert daily metric rollup rows."+ [rows]+ (chunked-insert! :model/SourceMetricDaily rows)+ nil)++(defn insert-dimension-rollups!+ "Insert daily dimension rollup rows."+ [rows]+ (chunked-insert! :model/SourceDimensionDaily rows)+ nil)++(defn insert-dimension-profile-rollups!+ "Insert daily dimension profile observation rows."+ [rows]+ (chunked-insert! :model/SourceDimensionProfileDaily rows)+ nil)++(defn replace-day!+ "Replace all rollup rows for `bucket-date` in one transaction."+ [bucket-date {:keys [segments composites metrics dimensions profiles]}]+ (t2/with-transaction [_conn]+ (delete-day! bucket-date)+ (insert-segment-rollups! segments)+ (insert-composite-rollups! composites)+ (insert-metric-rollups! metrics)+ (insert-dimension-rollups! dimensions)+ (insert-dimension-profile-rollups! profiles))+ nil)diff --git a/src/metabase/usage_metadata/task/process.clj b/src/metabase/usage_metadata/task/process.cljnew file mode 100644index 00000000000..a5e51c87118--- /dev/null+++ b/src/metabase/usage_metadata/task/process.clj@@ -0,0 +1,53 @@+(ns metabase.usage-metadata.task.process+ (:require+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.schedule.cron :as cron]+ [clojurewerkz.quartzite.triggers :as triggers]+ [metabase.task.core :as task]+ [metabase.usage-metadata.batch :as usage-metadata.batch]+ [metabase.usage-metadata.settings :as usage-metadata.settings]+ [metabase.util.log :as log])+ (:import+ (java.util TimeZone)))++(set! *warn-on-reflection* true)++(defn- cron-schedule+ "Bucket dates are always UTC days; fire the cron in UTC so the schedule hour matches the UTC day we'll close out."+ [cron-spec]+ (cron/schedule+ (cron/cron-schedule cron-spec)+ (cron/in-time-zone (TimeZone/getTimeZone "UTC"))+ (cron/with-misfire-handling-instruction-do-nothing)))++(def ^:private job-key+ (jobs/key "metabase.task.usage-metadata-process.job"))++(def ^:private trigger-key+ (triggers/key "metabase.task.usage-metadata-process.trigger"))++(task/defjob ^{org.quartz.DisallowConcurrentExecution true+ :doc "Process usage metadata rollups from query execution history."}+ UsageMetadataProcess+ [_]+ (when (usage-metadata.settings/usage-metadata-enabled?)+ (try+ (usage-metadata.batch/run-batch!)+ (catch Throwable e+ (log/error e "Error processing usage metadata batch")+ (throw e)))))++(defn- job []+ (jobs/build+ (jobs/of-type UsageMetadataProcess)+ (jobs/with-identity job-key)))++(defn- trigger []+ (triggers/build+ (triggers/with-identity trigger-key)+ (triggers/start-now)+ (triggers/with-schedule+ (cron-schedule (usage-metadata.settings/usage-metadata-schedule)))))++(defmethod task/init! ::UsageMetadataProcess [_]+ (task/schedule-task! (job) (trigger)))diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.cljindex 8df396d1969..f32bf8fd1b0 100644--- a/test/metabase/cmd/copy_test.clj+++ b/test/metabase/cmd/copy_test.clj@@ -57,6 +57,11 @@:model/SearchIndexMetadata:model/SecurityAdvisory:model/SemanticSearchTokenTracking+ :model/SourceDimensionDaily+ :model/SourceDimensionProfileDaily+ :model/SourceMetricDaily+ :model/SourceSegmentCompositeDaily+ :model/SourceSegmentDaily:model/SupportAccessGrantLog:model/TaskHistory:model/TaskRundiff --git a/test/metabase/lib/normalize_test.cljc b/test/metabase/lib/normalize_test.cljcindex a3a29ddf9c5..2d5b0f6bcfb 100644--- a/test/metabase/lib/normalize_test.cljc+++ b/test/metabase/lib/normalize_test.cljc@@ -104,6 +104,13 @@"aggregation" [["count" {}]]"filters" [["=" {} ["field" {} 1] 4]]}]})))))+(deftest ^:parallel lazy-seq-input-test+ (testing "Seq input (e.g. LazySeqs from json/decode+kw) normalizes"+ (is (=? [:= {:lib/uuid string?} [:field {:lib/uuid string?} 1] 4]+ (lib/normalize '("=" {} ("field" {} 1) 4))))+ (is (=? [:> {:lib/uuid string?} [:field {:lib/uuid string?} 1] 0]+ (lib/normalize '(">" {} ("field" {} 1) 0))))))+(deftest ^:parallel normalize-from-json-test(let [query '{:lib/type "mbql/query":lib/metadata nildiff --git a/test/metabase/metrics/api_test.clj b/test/metabase/metrics/api_test.cljindex 7f1bce6e53e..55e6b2f9074 100644--- a/test/metabase/metrics/api_test.clj+++ b/test/metabase/metrics/api_test.clj@@ -4,6 +4,7 @@[clojure.test :refer :all][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.query-processor.util :as qp.util][metabase.test :as mt][metabase.test.fixtures :as fixtures][toucan2.core :as t2]))@@ -332,3 +333,74 @@{:definition {:expression [:metric {:lib/uuid "a"} (:id metric)]:projections []}})](is (= "completed" (:status response)))))))++;;; +----------------------------------------------------------------------------------------------------------------++;;; | QueryExecution tracking |+;;; +----------------------------------------------------------------------------------------------------------------+++(defn- rasta-metric-executions+ "Return `QueryExecution` rows owned by the `:rasta` user, newest first.+ Filtering by executor keeps us from picking up unrelated rows written by+ concurrent tests."+ []+ (t2/select :model/QueryExecution+ :executor_id (mt/user->id :rasta)+ {:order-by [[:started_at :desc]]}))++(deftest dataset-leaf-records-query-execution-test+ (testing "POST /api/metric/dataset (leaf path) writes a QueryExecution row with :context :metric"+ (mt/test-helpers-set-global-values!+ (binding [qp.util/*execute-async?* false]+ (mt/with-temp [:model/Card metric {:name "QE Leaf Metric"+ :type :metric+ :dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]+ (let [before (count (rasta-metric-executions))]+ (mt/user-http-request :rasta :post 202 "metric/dataset"+ {:definition {:expression [:metric {:lib/uuid "a"} (:id metric)]}})+ (let [rows (rasta-metric-executions)]+ (is (= (inc before) (count rows))+ "exactly one new QueryExecution row for rasta")+ (is (= :metric (:context (first rows)))+ "the new row is tagged :context :metric"))))))))++(deftest dataset-arithmetic-records-one-qe-per-leaf-test+ (testing "POST /api/metric/dataset (arithmetic) writes one QueryExecution row per leaf"+ (mt/test-helpers-set-global-values!+ (binding [qp.util/*execute-async?* false]+ (mt/with-temp [:model/Card metric-a {:name "QE Arith A"+ :type :metric+ :dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}+ :model/Card metric-b {:name "QE Arith B"+ :type :metric+ :dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]+ (let [before (count (rasta-metric-executions))]+ (mt/user-http-request :rasta :post 202 "metric/dataset"+ {:definition {:expression [:+ {}+ [:metric {:lib/uuid "a"} (:id metric-a)]+ [:metric {:lib/uuid "b"} (:id metric-b)]]}})+ (let [rows (rasta-metric-executions)+ new-rows (take (- (count rows) before) rows)]+ (is (= 2 (count new-rows))+ "two new QueryExecution rows, one per arithmetic leaf")+ (is (every? #(= :metric (:context %)) new-rows)+ "both rows are tagged :context :metric"))))))))++(deftest breakout-values-records-query-execution-test+ (testing "POST /api/metric/breakout-values writes a QueryExecution row with :context :metric"+ (mt/test-helpers-set-global-values!+ (binding [qp.util/*execute-async?* false]+ (mt/with-temp [:model/Card metric {:name "QE Breakout Metric"+ :type :metric+ :dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]+ (mt/user-http-request :rasta :get 200 (str "metric/" (:id metric)))+ (let [dim-uuid (-> (mt/user-http-request :rasta :get 200 (str "metric/" (:id metric)))+ :dimensions first :id)+ before (count (rasta-metric-executions))]+ (mt/user-http-request :rasta :post 200 "metric/breakout-values"+ {:definition {:expression [:metric {:lib/uuid "a"} (:id metric)]+ :projections [{:type :metric :id (:id metric) :lib/uuid "a"+ :projection [[:dimension {} dim-uuid]]}]}})+ (let [rows (rasta-metric-executions)]+ (is (= (inc before) (count rows))+ "exactly one new QueryExecution row")+ (is (= :metric (:context (first rows)))))))))))diff --git a/test/metabase/usage_metadata/batch_test.clj b/test/metabase/usage_metadata/batch_test.cljnew file mode 100644index 00000000000..2e28965e85f--- /dev/null+++ b/test/metabase/usage_metadata/batch_test.clj@@ -0,0 +1,392 @@+(ns metabase.usage-metadata.batch-test+ (:require+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib-be.hash :as lib-be.hash]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [metabase.usage-metadata.batch :as usage-metadata.batch]+ [metabase.usage-metadata.models.source-dimension-daily]+ [metabase.usage-metadata.models.source-dimension-profile-daily]+ [metabase.usage-metadata.models.source-metric-daily]+ [metabase.usage-metadata.models.source-segment-composite-daily]+ [metabase.usage-metadata.models.source-segment-daily]+ [metabase.usage-metadata.settings :as usage-metadata.settings]+ [metabase.usage-metadata.store :as usage-metadata.store]+ [metabase.warehouse-schema.models.field]+ [toucan2.core :as t2])+ (:import+ (java.nio.charset StandardCharsets)))++(set! *warn-on-reflection* true)++(use-fixtures :once (fixtures/initialize :db))++(def ^:private execution-defaults+ {:running_time 1+ :result_rows 1+ :native false+ :executor_id nil+ :card_id nil+ :context :ad-hoc})++(defn- orders-query []+ (let [mp (lib-be/application-database-metadata-provider (mt/id))+ orders (lib.metadata/table mp (mt/id :orders))+ product-id (lib.metadata/field mp (mt/id :orders :product_id))+ subtotal (lib.metadata/field mp (mt/id :orders :subtotal))+ created-at (lib.metadata/field mp (mt/id :orders :created_at))+ user-id (lib.metadata/field mp (mt/id :orders :user_id))]+ (-> (lib/query mp orders)+ (lib/filter (lib/= product-id 1))+ (lib/aggregate (lib/sum subtotal))+ (lib/breakout (lib/with-temporal-bucket created-at :month))+ (lib/breakout user-id))))++(defn- cross-source-query []+ (let [mp (lib-be/application-database-metadata-provider (mt/id))+ venues (lib.metadata/table mp (mt/id :venues))+ categories (lib.metadata/table mp (mt/id :categories))+ venues-cat (lib.metadata/field mp (mt/id :venues :category_id))+ cat-id (lib.metadata/field mp (mt/id :categories :id))+ cat-id-join (lib/with-join-alias cat-id "Cat")]+ (-> (lib/query mp venues)+ (lib/join (-> (lib/join-clause categories [(lib/= venues-cat cat-id-join)])+ (lib/with-join-alias "Cat")))+ (lib/filter (lib/= venues-cat cat-id-join))+ (lib/breakout (lib.metadata/field mp (mt/id :venues :price))))))++(defn- native-query []+ {:database (mt/id)+ :type :native+ :native {:query "SELECT 1"}})++(defn- insert-query! [query-hash query]+ (t2/insert! :model/Query+ {:query_hash query-hash+ :query query+ :average_execution_time 1}))++(defn- insert-query-execution! [query-hash started-at]+ (t2/insert-returning-instances! :model/QueryExecution+ (assoc execution-defaults+ :hash query-hash+ :started_at started-at)))++(defn- delete-query! [query-hash]+ (t2/delete! :model/Query :query_hash query-hash))++(defn- delete-day! [bucket-date]+ (usage-metadata.store/delete-day! bucket-date))++(defn- delete-query-executions-for-day! [bucket-date]+ (t2/delete! :model/QueryExecution+ {:where [:and+ [:>= :started_at (t/offset-date-time bucket-date (t/local-time 0) (t/zone-offset "Z"))]+ [:< :started_at (t/offset-date-time (t/plus bucket-date (t/days 1)) (t/local-time 0) (t/zone-offset "Z"))]]}))++(deftest target-days-test+ (is (= [(t/local-date "2026-04-13")+ (t/local-date "2026-04-14")]+ (usage-metadata.batch/target-days+ {:last-completed-day (t/local-date "2026-04-12")+ :retention-days 90+ :yesterday (t/local-date "2026-04-14")})))+ (is (= [(t/local-date "2026-04-13")+ (t/local-date "2026-04-14")]+ (usage-metadata.batch/target-days+ {:last-completed-day nil+ :retention-days 2+ :yesterday (t/local-date "2026-04-14")})))+ (is (= [(t/local-date "2026-04-13")+ (t/local-date "2026-04-14")]+ (usage-metadata.batch/target-days+ {:last-completed-day (t/local-date "2026-04-01")+ :retention-days 2+ :yesterday (t/local-date "2026-04-14")}))))++(deftest process-day!-integration-test+ (let [bucket-date (t/local-date "2026-04-13")+ valid-query (orders-query)+ query-hash (lib-be.hash/query-hash valid-query)+ execution-at (t/offset-date-time "2026-04-13T12:00Z")+ user-id-field (mt/id :orders :user_id)+ original-fp (t2/select-one-fn :fingerprint :model/Field :id user-id-field)]+ (try+ (delete-query-executions-for-day! bucket-date)+ (delete-day! bucket-date)+ (t2/update! :model/Field :id user-id-field {:fingerprint {:global {:distinct-count 1, :nil% 0.0}}})+ (insert-query! query-hash valid-query)+ (insert-query-execution! query-hash execution-at)+ (insert-query-execution! query-hash (t/offset-date-time "2026-04-13T13:00Z"))+ (let [result (usage-metadata.batch/process-day! bucket-date)]+ (is (= {} (:skipped-rows result)))+ (is (= 2 (:query-execution-rows result)))+ (is (= 2 (:joined-rows result)))+ (is (= 2 (:segment-tuples result)))+ (is (= 2 (:metric-tuples result)))+ (is (= 4 (:dimension-tuples result)))+ (is (= 1 (:profile-observations result)))+ (is (= 1 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))+ (is (= 1 (t2/count :model/SourceMetricDaily :bucket_date bucket-date)))+ (is (= 2 (t2/count :model/SourceDimensionDaily :bucket_date bucket-date)))+ (is (= 1 (t2/count :model/SourceDimensionProfileDaily :bucket_date bucket-date)))+ (is (= :fingerprint+ (t2/select-one-fn :source_basis :model/SourceDimensionProfileDaily :bucket_date bucket-date)))+ (is (= :single-value+ (t2/select-one-fn :observation_type :model/SourceDimensionProfileDaily :bucket_date bucket-date)))+ (is (= "2026-04-13"+ (usage-metadata.settings/usage-metadata-last-completed-day))))+ (finally+ (t2/update! :model/Field :id user-id-field {:fingerprint original-fp})+ (delete-query! query-hash)+ (delete-query-executions-for-day! bucket-date)+ (delete-day! bucket-date)))))++(defn- composite-orders-query []+ (let [mp (lib-be/application-database-metadata-provider (mt/id))+ orders (lib.metadata/table mp (mt/id :orders))+ product-id (lib.metadata/field mp (mt/id :orders :product_id))+ subtotal (lib.metadata/field mp (mt/id :orders :subtotal))]+ (-> (lib/query mp orders)+ (lib/filter (lib/and (lib/= product-id 1)+ (lib/> subtotal 0)))+ (lib/aggregate (lib/count)))))++(deftest process-day!-composite-segment-test+ (testing "a top-level :and filter writes both atom rollup rows AND one composite rollup row"+ (let [bucket-date (t/local-date "2026-04-13")+ valid-query (composite-orders-query)+ query-hash (lib-be.hash/query-hash valid-query)+ execution-at (t/offset-date-time "2026-04-13T12:00Z")]+ (try+ (delete-query-executions-for-day! bucket-date)+ (delete-day! bucket-date)+ (insert-query! query-hash valid-query)+ (insert-query-execution! query-hash execution-at)+ (let [result (usage-metadata.batch/process-day! bucket-date)]+ (is (= 2 (:segment-tuples result)))+ (is (= 1 (:composite-tuples result))+ "one composite fact per execution for the two-atom :and clause")+ (is (= 2 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))+ (is (= 1 (t2/count :model/SourceSegmentCompositeDaily :bucket_date bucket-date)))+ (let [composite (t2/select-one :model/SourceSegmentCompositeDaily :bucket_date bucket-date)]+ (is (= :direct (:ownership_mode composite)))+ (is (= 2 (:atom_count composite)))+ (is (= 1 (:count composite)))))+ (finally+ (delete-query! query-hash)+ (delete-query-executions-for-day! bucket-date)+ (delete-day! bucket-date))))))++(deftest process-day!-cross-source-predicate-test+ (let [bucket-date (t/local-date "2026-04-13")+ valid-query (cross-source-query)+ query-hash (lib-be.hash/query-hash valid-query)+ execution-at (t/offset-date-time "2026-04-13T12:00Z")]+ (try+ (delete-query-executions-for-day! bucket-date)+ (delete-day! bucket-date)+ (insert-query! query-hash valid-query)+ (insert-query-execution! query-hash execution-at)+ (usage-metadata.batch/process-day! bucket-date)+ (is (= #{:mixed :projected}+ (t2/select-fn-set :ownership_mode :model/SourceSegmentDaily :bucket_date bucket-date)))+ (is (= 3 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))+ (finally+ (delete-query! query-hash)+ (delete-query-executions-for-day! bucket-date)+ (delete-day! bucket-date)))))++(deftest reprocess-day!-does-not-advance-watermark-test+ (let [bucket-date (t/local-date "2026-04-13")+ valid-query (orders-query)+ query-hash (lib-be.hash/query-hash valid-query)+ execution-at (t/offset-date-time "2026-04-13T12:00Z")]+ (try+ (delete-query-executions-for-day! bucket-date)+ (delete-day! bucket-date)+ (insert-query! query-hash valid-query)+ (insert-query-execution! query-hash execution-at)+ (mt/with-temporary-setting-values [usage-metadata-last-completed-day "2026-04-12"]+ (let [result (usage-metadata.batch/reprocess-day! bucket-date)]+ (is (false? (:watermark-advanced? result)))+ (is (= "2026-04-12"+ (usage-metadata.settings/usage-metadata-last-completed-day)))+ (is (= 1 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))))+ (finally+ (delete-query! query-hash)+ (delete-query-executions-for-day! bucket-date)+ (delete-day! bucket-date)))))++(deftest run-batch!-integration-test+ (let [day-a (t/local-date "2026-04-13")+ day-b (t/local-date "2026-04-14")+ old-day (t/local-date "2026-04-10")+ valid-query (orders-query)+ valid-hash (lib-be.hash/query-hash valid-query)+ native-hash (lib-be.hash/query-hash (native-query))+ missing-hash (.getBytes "missing-query" StandardCharsets/UTF_8)+ unsupported-hash (.getBytes "unsupported-query" StandardCharsets/UTF_8)]+ (try+ (doseq [day [day-a day-b old-day]]+ (delete-query-executions-for-day! day)+ (delete-day! day))+ (insert-query! valid-hash valid-query)+ (insert-query! native-hash (native-query))+ (insert-query! unsupported-hash {})+ (insert-query-execution! valid-hash (t/offset-date-time "2026-04-13T12:00Z"))+ (insert-query-execution! valid-hash (t/offset-date-time "2026-04-14T12:00Z"))+ (insert-query-execution! native-hash (t/offset-date-time "2026-04-14T13:00Z"))+ (insert-query-execution! missing-hash (t/offset-date-time "2026-04-14T14:00Z"))+ (insert-query-execution! unsupported-hash (t/offset-date-time "2026-04-14T15:00Z"))+ (t2/insert! :model/SourceSegmentDaily+ {:source_type :table+ :source_id 1+ :ownership_mode :direct+ :field_id 1+ :predicate "old"+ :bucket_date old-day+ :count 1})+ (mt/with-temporary-setting-values [usage-metadata-last-completed-day nil+ usage-metadata-retention-days 2]+ (let [summary (usage-metadata.batch/run-batch!+ {:today (t/local-date "2026-04-15")+ :last-completed-day nil+ :retention-days 2})]+ (is (= :success (:status summary)))+ (is (= 2 (:days-processed summary)))+ (is (= 5 (:query-execution-rows summary)))+ (is (= 4 (:joined-rows summary)))+ (is (= 1 (get-in summary [:skipped-rows :missing-query])))+ (is (= 1 (get-in summary [:skipped-rows :unsupported-query])))+ (is (= 1+ (t2/count :model/SourceSegmentDaily :bucket_date day-a)))+ (is (= 1+ (t2/count :model/SourceSegmentDaily :bucket_date day-b)))+ (is (= 0+ (t2/count :model/SourceSegmentDaily :bucket_date old-day)))+ (is (= "2026-04-14"+ (usage-metadata.settings/usage-metadata-last-completed-day)))))+ (finally+ (delete-query! valid-hash)+ (delete-query! native-hash)+ (delete-query! unsupported-hash)+ (doseq [day [day-a day-b old-day]]+ (delete-query-executions-for-day! day)+ (delete-day! day))))))++(deftest run-batch!-retention-runs-even-on-failure-test+ (let [day-a (t/local-date "2026-04-13")+ day-b (t/local-date "2026-04-14")+ old-day (t/local-date "2026-04-10")+ valid-query (orders-query)+ query-hash (lib-be.hash/query-hash valid-query)+ original usage-metadata.store/replace-day!]+ (try+ (doseq [day [day-a day-b old-day]]+ (delete-query-executions-for-day! day)+ (delete-day! day))+ (insert-query! query-hash valid-query)+ (insert-query-execution! query-hash (t/offset-date-time "2026-04-13T12:00Z"))+ (insert-query-execution! query-hash (t/offset-date-time "2026-04-14T12:00Z"))+ (t2/insert! :model/SourceSegmentDaily+ {:source_type :table+ :source_id 1+ :ownership_mode :direct+ :field_id 1+ :predicate "old"+ :bucket_date old-day+ :count 1})+ (mt/with-temporary-setting-values [usage-metadata-last-completed-day nil+ usage-metadata-retention-days 2]+ (with-redefs [usage-metadata.store/replace-day! (fn [bucket-date payload]+ (if (= bucket-date day-b)+ (throw (ex-info "boom" {:bucket-date bucket-date}))+ (original bucket-date payload)))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (usage-metadata.batch/run-batch!+ {:today (t/local-date "2026-04-15")+ :last-completed-day nil+ :retention-days 2})))+ (testing "retention cleanup runs even when the processing loop fails"+ (is (= 0 (t2/count :model/SourceSegmentDaily :bucket_date old-day))))))+ (finally+ (delete-query! query-hash)+ (doseq [day [day-a day-b old-day]]+ (delete-query-executions-for-day! day)+ (delete-day! day))))))++(deftest process-day!-skips-rows-when-normalize-query-throws-test+ (testing "a row whose query throws inside normalize-query is counted as :normalize-error+ and does not block the rest of the day from rolling up"+ (let [bucket-date (t/local-date "2026-04-13")+ good-query (orders-query)+ good-hash (lib-be.hash/query-hash good-query)+ bad-hash (.getBytes "bad-query" StandardCharsets/UTF_8)+ bad-payload {:cooked :poison}+ original lib-be/normalize-query]+ (try+ (delete-query-executions-for-day! bucket-date)+ (delete-day! bucket-date)+ (insert-query! good-hash good-query)+ (insert-query! bad-hash bad-payload)+ (insert-query-execution! good-hash (t/offset-date-time "2026-04-13T12:00Z"))+ (insert-query-execution! bad-hash (t/offset-date-time "2026-04-13T13:00Z"))+ (with-redefs [lib-be/normalize-query (fn [stored-query]+ (if (= stored-query bad-payload)+ (throw (ex-info "kaboom" {:q stored-query}))+ (original stored-query)))]+ (mt/with-temporary-setting-values [usage-metadata-last-completed-day "2026-04-12"]+ (let [result (usage-metadata.batch/process-day! bucket-date)]+ (testing "the bad row is recorded as :normalize-error skip"+ (is (= 1 (get-in result [:skipped-rows :normalize-error]))))+ (testing "the good row's rollups still landed"+ (is (= 1 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))+ (is (= 1 (t2/count :model/SourceMetricDaily :bucket_date bucket-date))))+ (testing "the watermark advances despite the per-row failure"+ (is (true? (:watermark-advanced? result)))+ (is (= "2026-04-13"+ (usage-metadata.settings/usage-metadata-last-completed-day)))))))+ (finally+ (delete-query! good-hash)+ (delete-query! bad-hash)+ (delete-query-executions-for-day! bucket-date)+ (delete-day! bucket-date))))))++(deftest run-batch!-stops-on-day-failure-test+ (let [day-a (t/local-date "2026-04-13")+ day-b (t/local-date "2026-04-14")+ valid-query (orders-query)+ query-hash (lib-be.hash/query-hash valid-query)+ original usage-metadata.store/replace-day!]+ (try+ (doseq [day [day-a day-b]]+ (delete-query-executions-for-day! day)+ (delete-day! day))+ (insert-query! query-hash valid-query)+ (insert-query-execution! query-hash (t/offset-date-time "2026-04-13T12:00Z"))+ (insert-query-execution! query-hash (t/offset-date-time "2026-04-14T12:00Z"))+ (mt/with-temporary-setting-values [usage-metadata-last-completed-day nil+ usage-metadata-retention-days 2]+ (with-redefs [usage-metadata.store/replace-day! (fn [bucket-date payload]+ (if (= bucket-date day-b)+ (throw (ex-info "boom" {:bucket-date bucket-date}))+ (original bucket-date payload)))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (usage-metadata.batch/run-batch!+ {:today (t/local-date "2026-04-15")+ :last-completed-day nil+ :retention-days 2})))+ (is (= 1 (t2/count :model/SourceSegmentDaily :bucket_date day-a)))+ (is (= 0 (t2/count :model/SourceSegmentDaily :bucket_date day-b)))+ (is (= "2026-04-13"+ (usage-metadata.settings/usage-metadata-last-completed-day)))))+ (finally+ (delete-query! query-hash)+ (doseq [day [day-a day-b]]+ (delete-query-executions-for-day! day)+ (delete-day! day))))))diff --git a/test/metabase/usage_metadata/core_test.clj b/test/metabase/usage_metadata/core_test.cljnew file mode 100644index 00000000000..d54962ce6ba--- /dev/null+++ b/test/metabase/usage_metadata/core_test.clj@@ -0,0 +1,90 @@+(ns metabase.usage-metadata.core-test+ (:require+ [clojure.test :refer :all]+ [metabase.usage-metadata.core :as usage-metadata]+ [metabase.usage-metadata.insights :as insights]))++(def ^:private sample-segment+ {:predicate [:= [:field 1 nil] 1]+ :source {:type :card, :id 99, :name "Q", :display-name "Q"}+ :fields [{:id 1, :name "f", :display-name "F"}]+ :count 1})++(def ^:private sample-metric+ {:source {:type :table, :id 42, :name "T", :display-name "T", :db-id 1, :schema nil}+ :aggregation {:type :count, :field nil, :temporal-field nil, :temporal-unit nil}+ :count 1})++(def ^:private sample-dimension+ {:source {:type :table, :id 42, :name "T", :display-name "T", :db-id 1, :schema nil}+ :dimension {:field {:id 1, :name "f", :display-name "F"}, :temporal-unit nil, :binning nil}+ :count 1})++(def ^:private sample-suggested-segment+ {:clause [:and [:= [:field 1 nil] 1] [:> [:field 2 nil] 0]]+ :itemset-size 2+ :source {:type :table, :id 42, :name "T", :display-name "T", :db-id 1, :schema nil}+ :support 3+ :support-ratio 1.0})++(def ^:private sample-profile-observation+ {:source {:type :table, :id 42, :name "T", :display-name "T", :db-id 1, :schema nil}+ :field {:id 1, :name "f", :display-name "F"}+ :basis :fingerprint+ :observation {:type :low-cardinality, :value 5}+ :count 1})++(deftest implicit-segments-delegate-to-insights-test+ (let [captured-args (atom nil)]+ (with-redefs [insights/implicit-segments+ (fn [opts]+ (reset! captured-args opts)+ [sample-segment])]+ (is (= [sample-segment]+ (usage-metadata/implicit-segments {:source-type :card, :source-id 99, :limit 3})))+ (is (= {:source-type :card, :source-id 99, :limit 3}+ @captured-args)))))++(deftest implicit-metrics-delegate-to-insights-test+ (let [captured-args (atom nil)]+ (with-redefs [insights/implicit-metrics+ (fn [opts]+ (reset! captured-args opts)+ [sample-metric])]+ (is (= [sample-metric]+ (usage-metadata/implicit-metrics {:source-type :table, :source-id 42, :limit 7})))+ (is (= {:source-type :table, :source-id 42, :limit 7}+ @captured-args)))))++(deftest implicit-dimensions-delegate-to-insights-test+ (let [captured-args (atom nil)]+ (with-redefs [insights/implicit-dimensions+ (fn [opts]+ (reset! captured-args opts)+ [sample-dimension])]+ (is (= [sample-dimension]+ (usage-metadata/implicit-dimensions {:source-type :table, :source-id 42, :limit 9})))+ (is (= {:source-type :table, :source-id 42, :limit 9}+ @captured-args)))))++(deftest suggested-segments-delegate-to-insights-test+ (let [captured-args (atom nil)]+ (with-redefs [insights/suggested-segments-for-owner+ (fn [opts]+ (reset! captured-args opts)+ [sample-suggested-segment])]+ (is (= [sample-suggested-segment]+ (usage-metadata/suggested-segments {:source-type :table, :source-id 42, :limit 5})))+ (is (= {:source-type :table, :source-id 42, :limit 5}+ @captured-args)))))++(deftest profile-observations-delegate-to-insights-test+ (let [captured-args (atom nil)]+ (with-redefs [insights/profile-observations+ (fn [opts]+ (reset! captured-args opts)+ [sample-profile-observation])]+ (is (= [sample-profile-observation]+ (usage-metadata/profile-observations {:source-type :table, :source-id 42, :limit 4})))+ (is (= {:source-type :table, :source-id 42, :limit 4}+ @captured-args)))))diff --git a/test/metabase/usage_metadata/extract_test.clj b/test/metabase/usage_metadata/extract_test.cljnew file mode 100644index 00000000000..58f51360c46--- /dev/null+++ b/test/metabase/usage_metadata/extract_test.clj@@ -0,0 +1,239 @@+(ns metabase.usage-metadata.extract-test+ (:require+ [clojure.test :refer :all]+ [metabase.lib.convert :as lib.convert]+ [metabase.lib.core :as lib]+ [metabase.lib.test-metadata :as meta]+ [metabase.lib.test-util :as lib.tu]+ [metabase.usage-metadata.extract :as extract]+ [metabase.util.json :as json]))++(defn- decode-json+ [s]+ (json/decode+kw s))++(deftest ^:parallel query-source-table-or-card-test+ (testing "table roots are attributed to the table"+ (is (= {:source-type :table, :source-id (meta/id :venues)}+ (extract/query-source-table-or-card (lib.tu/venues-query)))))+ (testing "card roots are attributed to the card"+ (is (= {:source-type :card, :source-id 1}+ (extract/query-source-table-or-card (lib.tu/query-with-source-card)))))+ (testing "model roots are still attributed as cards"+ (is (= {:source-type :card, :source-id 1}+ (extract/query-source-table-or-card (lib.tu/query-with-source-model)))))+ (testing "native queries are skipped"+ (is (nil? (extract/query-source-table-or-card (lib.tu/native-query))))))++(deftest ^:parallel segment-extraction-test+ (let [query (-> (lib.tu/venues-query)+ (lib/filter (lib/= (meta/field-metadata :venues :price) 4))+ (lib/filter (lib/> (meta/field-metadata :venues :id) 10)))+ rows (:segments (extract/extract-usage-facts query))]+ (is (= 2 (count rows)))+ (is (= #{:direct} (set (map :ownership-mode rows))))+ (is (= #{{:source-type :table, :source-id (meta/id :venues), :field-id (meta/id :venues :price)}+ {:source-type :table, :source-id (meta/id :venues), :field-id (meta/id :venues :id)}}+ (set (map #(select-keys % [:source-type :source-id :field-id]) rows))))+ (is (= #{["=" 4] [">" 10]}+ (set (map (fn [{:keys [predicate]}]+ (let [decoded (decode-json predicate)]+ [(first decoded) (last decoded)]))+ rows))))))++(deftest ^:parallel compound-filter-extraction-test+ (let [query (lib/filter (lib.tu/venues-query)+ (lib/or (lib/= (meta/field-metadata :venues :price) 1)+ (lib/= (meta/field-metadata :venues :price) 2)))+ rows (:segments (extract/extract-usage-facts query))]+ (is (= 1 (count rows)))+ (is (= :direct (:ownership-mode (first rows))))+ (is (= (meta/id :venues :price) (:field-id (first rows))))+ (is (= "or"+ (first (decode-json (:predicate (first rows))))))))++(deftest ^:parallel metric-extraction-test+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/aggregate (lib/sum (meta/field-metadata :orders :subtotal)))+ (lib/breakout (lib/with-temporal-bucket (meta/field-metadata :orders :created-at) :month))+ (lib/breakout (meta/field-metadata :orders :product-id)))+ rows (:metrics (extract/extract-usage-facts query))]+ (is (= [{:source-type :table+ :source-id (meta/id :orders)+ :ownership-mode :direct+ :agg :sum+ :agg-field-id (meta/id :orders :subtotal)+ :temporal-field-id (meta/id :orders :created-at)+ :temporal-unit :month}]+ rows))))++(deftest ^:parallel metric-extraction-without-temporal-breakout-test+ (let [query (-> (lib.tu/venues-query)+ (lib/aggregate (lib/count)))+ rows (:metrics (extract/extract-usage-facts query))]+ (is (= [{:source-type :table+ :source-id (meta/id :venues)+ :ownership-mode :direct+ :agg :count+ :agg-field-id nil+ :temporal-field-id nil+ :temporal-unit nil}]+ rows))))++(deftest ^:parallel dimension-extraction-test+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/breakout (lib/with-temporal-bucket (meta/field-metadata :orders :created-at) :month))+ (lib/breakout (lib/with-binning (meta/field-metadata :orders :subtotal)+ {:strategy :num-bins, :num-bins 10})))+ rows (:dimensions (extract/extract-usage-facts query))]+ (is (= [{:source-type :table+ :source-id (meta/id :orders)+ :ownership-mode :direct+ :field-id (meta/id :orders :created-at)+ :temporal-unit :month+ :binning nil}+ {:source-type :table+ :source-id (meta/id :orders)+ :ownership-mode :direct+ :field-id (meta/id :orders :subtotal)+ :temporal-unit nil+ :binning "{\"num-bins\":10,\"strategy\":\"num-bins\"}"}]+ rows))+ (is (= {:num-bins 10, :strategy "num-bins"}+ (decode-json (:binning (second rows)))))))++(deftest ^:parallel joined-field-is-owned-by-joined-source-test+ (let [query (-> (lib.tu/query-with-join)+ (lib/filter (lib/= (meta/field-metadata :venues :price) 1))+ (lib/filter (lib/= (lib/with-join-alias (meta/field-metadata :categories :name) "Cat") "Pizza")))+ rows (:segments (extract/extract-usage-facts query))]+ (is (= #{{:source-type :table, :source-id (meta/id :venues), :ownership-mode :direct, :field-id (meta/id :venues :price)}+ {:source-type :table, :source-id (meta/id :categories), :ownership-mode :direct, :field-id (meta/id :categories :name)}}+ (set (map #(select-keys % [:source-type :source-id :ownership-mode :field-id]) rows))))))++(deftest ^:parallel source-card-fields-stay-card-owned-test+ (let [query (-> (lib.tu/query-with-source-model)+ (lib/append-stage)+ (lib/filter -1 (lib/= (meta/field-metadata :checkins :user-id) 10)))+ rows (:segments (extract/extract-usage-facts query))]+ (is (= 1 (count rows)))+ (is (= {:source-type :card, :source-id 1, :ownership-mode :direct, :field-id (meta/id :checkins :user-id)}+ (select-keys (first rows) [:source-type :source-id :ownership-mode :field-id])))))++(deftest ^:parallel composite-extraction-single-owner-test+ (testing "a top-level :and clause produces exactly one composite fact on a single-owner query"+ (let [query (lib/filter (lib.tu/venues-query)+ (lib/and (lib/= (meta/field-metadata :venues :price) 4)+ (lib/> (meta/field-metadata :venues :id) 10)))+ facts (:composites (extract/extract-usage-facts query))]+ (is (= 1 (count facts)))+ (let [{:keys [source-type source-id ownership-mode clause atom-fingerprints atom-count]} (first facts)]+ (is (= :table source-type))+ (is (= (meta/id :venues) source-id))+ (is (= :direct ownership-mode))+ (is (= 2 atom-count))+ (is (= 2 (count atom-fingerprints)))+ (is (= atom-fingerprints (vec (sort atom-fingerprints)))+ "atom-fingerprints should be sorted for stable basket identity")+ (is (= "and" (first (decode-json clause))))+ (testing "atom fingerprints round-trip to canonical atom JSON"+ (is (= #{"=" ">"} (set (map (comp first decode-json) atom-fingerprints)))))))))++(deftest ^:parallel composite-extraction-single-atom-is-skipped-test+ (testing "single-atom top-level filters emit NO composite (atom rollup already captures them)"+ (let [query (lib/filter (lib.tu/venues-query)+ (lib/= (meta/field-metadata :venues :price) 4))]+ (is (empty? (:composites (extract/extract-usage-facts query)))))))++(deftest ^:parallel composite-extraction-accumulated-filters-combine-into-one-basket-test+ (testing "two separate lib/filter calls form a single composite basket (the stage's implicit :and)"+ (let [query (-> (lib.tu/venues-query)+ (lib/filter (lib/= (meta/field-metadata :venues :price) 4))+ (lib/filter (lib/> (meta/field-metadata :venues :id) 10)))+ facts (:composites (extract/extract-usage-facts query))]+ (is (= 1 (count facts)))+ (is (= 2 (:atom-count (first facts)))))))++(deftest ^:parallel composite-extraction-multi-owner-test+ (testing ":and spanning multiple owners yields :mixed + per-owner :projected composite facts"+ (let [query (lib/filter (lib.tu/query-with-join)+ (lib/and (lib/= (meta/field-metadata :venues :price) 1)+ (lib/= (lib/with-join-alias (meta/field-metadata :categories :name) "Cat") "Pizza")))+ facts (:composites (extract/extract-usage-facts query))]+ (is (= 3 (count facts)))+ (is (= 1 (count (filter #(= :mixed (:ownership-mode %)) facts))))+ (is (= #{{:source-type :table, :source-id (meta/id :venues), :ownership-mode :projected}+ {:source-type :table, :source-id (meta/id :categories), :ownership-mode :projected}}+ (set (map #(select-keys % [:source-type :source-id :ownership-mode])+ (filter #(= :projected (:ownership-mode %)) facts)))))+ (testing "all projected rows share the same clause+atom fingerprints as the mixed row"+ (let [fingerprints (into #{} (map (juxt :clause :atom-fingerprints)) facts)]+ (is (= 1 (count fingerprints))))))))++(deftest ^:parallel cross-source-predicate-produces-mixed-and-projected-test+ (let [query (-> (lib.tu/query-with-join)+ (lib/filter (lib/= (meta/field-metadata :venues :category-id)+ (lib/with-join-alias (meta/field-metadata :categories :id) "Cat"))))+ rows (:segments (extract/extract-usage-facts query))]+ (is (= 3 (count rows)))+ (is (= 1 (count (filter #(= :mixed (:ownership-mode %)) rows))))+ (is (= #{{:source-type :table, :source-id (meta/id :venues), :ownership-mode :projected, :field-id (meta/id :venues :category-id)}+ {:source-type :table, :source-id (meta/id :categories), :ownership-mode :projected, :field-id (meta/id :categories :id)}}+ (set (map #(select-keys % [:source-type :source-id :ownership-mode :field-id])+ (filter #(= :projected (:ownership-mode %)) rows)))))))++(def ^:private metric-id-1 200)+(def ^:private metric-id-2 201)++(def ^:private metrics-metadata-provider+ (lib.tu/mock-metadata-provider+ meta/metadata-provider+ {:cards [{:id metric-id-1+ :name "Sum Subtotal"+ :type :metric+ :database-id (meta/id)+ :table-id (meta/id :orders)+ :dataset-query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/aggregate (lib/sum (meta/field-metadata :orders :subtotal)))+ lib.convert/->legacy-MBQL)}+ {:id metric-id-2+ :name "Sum Tax"+ :type :metric+ :database-id (meta/id)+ :table-id (meta/id :orders)+ :dataset-query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/aggregate (lib/sum (meta/field-metadata :orders :tax)))+ lib.convert/->legacy-MBQL)}]}))++(deftest ^:parallel metric-extraction-only-records-primitive-aggregations-test+ (testing "primitive Sum(field) aggregation emits a row"+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/aggregate (lib/sum (meta/field-metadata :orders :subtotal))))+ rows (:metrics (extract/extract-usage-facts query))]+ (is (= 1 (count rows)))+ (is (= :sum (:agg (first rows))))+ (is (= (meta/id :orders :subtotal) (:agg-field-id (first rows))))))+ (testing "top-level [:metric M1] reference emits no row (F001)"+ (let [query (-> (lib/query metrics-metadata-provider (meta/table-metadata :orders))+ (lib/aggregate [:metric {:lib/uuid (str (random-uuid))} metric-id-1]))+ rows (:metrics (extract/extract-usage-facts query))]+ (is (= [] rows))))+ (testing "composite [:- [:metric M1] [:metric M2]] emits no row (F015)"+ (let [query (-> (lib/query metrics-metadata-provider (meta/table-metadata :orders))+ (lib/aggregate (lib/- [:metric {:lib/uuid (str (random-uuid))} metric-id-1]+ [:metric {:lib/uuid (str (random-uuid))} metric-id-2])))+ rows (:metrics (extract/extract-usage-facts query))]+ (is (= [] rows))))+ (testing "composite Sum(A) - Sum(B) emits no row (F015)"+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/aggregate (lib/- (lib/sum (meta/field-metadata :orders :subtotal))+ (lib/sum (meta/field-metadata :orders :tax)))))+ rows (:metrics (extract/extract-usage-facts query))]+ (is (= [] rows))))+ (testing "Sum(coalesce(A, 0)) emits a row with field A captured"+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/aggregate (lib/sum (lib/coalesce (meta/field-metadata :orders :subtotal) 0))))+ rows (:metrics (extract/extract-usage-facts query))]+ (is (= 1 (count rows)))+ (is (= :sum (:agg (first rows))))+ (is (= (meta/id :orders :subtotal) (:agg-field-id (first rows)))))))diff --git a/test/metabase/usage_metadata/insights_test.clj b/test/metabase/usage_metadata/insights_test.cljnew file mode 100644index 00000000000..34408b973dd--- /dev/null+++ b/test/metabase/usage_metadata/insights_test.clj@@ -0,0 +1,248 @@+(ns metabase.usage-metadata.insights-test+ (:require+ [clojure.core.memoize :as memoize]+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [metabase.usage-metadata.extract :as usage-metadata.extract]+ [metabase.usage-metadata.insights :as insights]+ [metabase.usage-metadata.models.source-segment-composite-daily]+ [metabase.util.json :as json]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db))++(def ^:private mine-itemsets @#'insights/mine-itemsets)+(def ^:private closed-only @#'insights/closed-only)+(def ^:private itemset-support @#'insights/itemset-support)+(def ^:private any-atom-support @#'insights/any-atom-support)+(def ^:private rebuild-and-clause @#'insights/rebuild-and-clause)+(def ^:private relative-support-ok? @#'insights/relative-support-ok?)+(def ^:private existing-composite-atomsets @#'insights/existing-composite-atomsets)+(def ^:private composite-atomsets-memo @#'insights/existing-composite-atomsets*-memo)++(deftest ^:parallel itemset-support-counts-containing-baskets-weighted-by-count-test+ (let [baskets [{:atoms #{:a :b :c} :count 3}+ {:atoms #{:a :b} :count 2}+ {:atoms #{:a :c} :count 1}+ {:atoms #{:b} :count 4}]]+ (testing "support = sum of basket counts for baskets containing ALL atoms in the itemset"+ (is (= 5 (itemset-support baskets #{:a :b})))+ (is (= 4 (itemset-support baskets #{:a :c})))+ (is (= 3 (itemset-support baskets #{:a :b :c})))+ (is (= 0 (itemset-support baskets #{:z}))))+ (testing "any-atom-support = sum of basket counts for baskets containing ANY atom in the itemset"+ (is (= 10 (any-atom-support baskets #{:a :b})))+ (is (= 6 (any-atom-support baskets #{:a}))))+ (testing "a basket contributes to an itemset's support exactly once per its own count"+ (doseq [basket baskets]+ (let [contributes? (every? (:atoms basket) #{:a :b})]+ (is (= (if contributes? (:count basket) 0)+ (itemset-support [basket] #{:a :b}))))))))++(deftest ^:parallel mine-itemsets-worked-example-test+ (testing "design-doc worked example: baskets {a1,a2,a3}+{a1,a2,a4} yield the {a1,a2} closed itemset"+ (let [baskets [{:atoms #{:a1 :a2 :a3} :count 1}+ {:atoms #{:a1 :a2 :a4} :count 1}]+ mined (mine-itemsets baskets)+ closed (closed-only mined)]+ (is (= {[:a1 :a2] 2} mined)+ "absolute support floor of 2 keeps only the pair that co-occurs in both baskets")+ (is (= {[:a1 :a2] 2} closed)+ "closed filter has nothing to collapse when only one itemset survives"))))++(deftest ^:parallel closed-only-drops-subsumed-itemsets-of-equal-support-test+ (testing "if {a,b} and {a,b,c} have equal support, closed filter drops the smaller subset"+ (is (= {[:a :b :c] 2}+ (closed-only {[:a :b] 2+ [:a :b :c] 2}))))+ (testing "subsets with strictly greater support are preserved"+ (is (= {[:a :b] 5+ [:a :b :c] 2}+ (closed-only {[:a :b] 5+ [:a :b :c] 2}))))+ (testing "unrelated itemsets are untouched"+ (is (= {[:a :b] 3 [:c :d] 3}+ (closed-only {[:a :b] 3 [:c :d] 3})))))++(deftest ^:parallel mine-itemsets-respects-size-bounds-test+ (testing "only itemsets of size ≥ fim-k-min (2) appear in output — no singletons"+ (let [baskets [{:atoms #{:a :b} :count 5}]+ mined (mine-itemsets baskets)]+ (is (every? #(>= (count %) 2) (keys mined)))+ (is (not (contains? mined [:a])))+ (is (not (contains? mined [:b]))))))++(deftest existing-segment-predicates-cached-test+ (testing "existing-segment-predicates is TTL-memoized — repeated calls with the same opts hit the DB once"+ (let [segment-selects (atom 0)+ real-select t2/select+ existing-fn @#'insights/existing-segment-predicates+ memo-var @#'insights/existing-segment-predicates*-memo]+ (memoize/memo-clear! memo-var)+ (with-redefs [t2/select (fn [& args]+ (when (and (sequential? (first args))+ (= :model/Segment (ffirst args)))+ (swap! segment-selects inc))+ (apply real-select args))]+ (existing-fn {})+ (existing-fn {})+ (is (= 1 @segment-selects)))+ (memoize/memo-clear! memo-var))))++(deftest existing-metric-signatures-cached-test+ (testing "existing-metric-signatures is TTL-memoized"+ (let [card-selects (atom 0)+ real-select t2/select+ existing-fn @#'insights/existing-metric-signatures+ memo-var @#'insights/existing-metric-signatures*-memo]+ (memoize/memo-clear! memo-var)+ (with-redefs [t2/select (fn [& args]+ (when (and (sequential? (first args))+ (= :model/Card (ffirst args)))+ (swap! card-selects inc))+ (apply real-select args))]+ (existing-fn)+ (existing-fn)+ (is (= 1 @card-selects)))+ (memoize/memo-clear! memo-var))))++;;; ---------- composite-segment read-path tests ----------++(deftest ^:parallel rebuild-and-clause-test+ (let [fp-a "[\"=\",{},[\"field\",{},1],1]"+ fp-b "[\">\",{},[\"field\",{},2],0]"]+ (testing "builds a properly-shaped :and MBQL clause from atom fingerprints"+ (is (lib/clause-of-type? (rebuild-and-clause [fp-a fp-b]) :and)))+ (testing "returns nil below fim-k-min (2) atoms"+ (is (nil? (rebuild-and-clause [])))+ (is (nil? (rebuild-and-clause [fp-a]))))+ (testing "returns nil when decode-predicate drops everything below the floor"+ (is (nil? (rebuild-and-clause [nil nil])))+ (is (nil? (rebuild-and-clause [fp-a nil]))))))++(deftest ^:parallel relative-support-ok?-test+ (testing "ratio above 0.2 floor → true"+ (let [baskets [{:atoms #{:a :b} :count 4}+ {:atoms #{:a :c} :count 1}]]+ (is (relative-support-ok? baskets [:a :b] 4))))+ (testing "ratio below 0.2 floor → false"+ (let [baskets [{:atoms #{:a :b} :count 1}+ {:atoms #{:a :c} :count 4}+ {:atoms #{:b :c} :count 5}]]+ (is (not (relative-support-ok? baskets [:a :b] 1)))))+ (testing "zero denominator → true (boundary)"+ (is (relative-support-ok? [] [:z] 0))))++(def ^:private composite-test-bucket-date (t/local-date "2099-01-01"))++(defn- composite-orders-query []+ (let [mp (lib-be/application-database-metadata-provider (mt/id))+ orders (lib.metadata/table mp (mt/id :orders))+ prod-id (lib.metadata/field mp (mt/id :orders :product_id))+ subtotal (lib.metadata/field mp (mt/id :orders :subtotal))]+ (-> (lib/query mp orders)+ (lib/filter (lib/and (lib/= prod-id 1)+ (lib/> subtotal 0))))))++(defn- composite-fact-for-orders []+ (->> (composite-orders-query)+ usage-metadata.extract/extract-usage-facts+ :composites+ (filter (fn [{:keys [source-type ownership-mode]}]+ (and (= :table source-type) (= :direct ownership-mode))))+ first))++(defn- seed-composite-row!+ [{:keys [source-type source-id clause atom-fingerprints atom-count]} cnt]+ (t2/insert! :model/SourceSegmentCompositeDaily+ {:source_type source-type+ :source_id source-id+ :ownership_mode :direct+ :clause clause+ :atom_fingerprints (json/encode atom-fingerprints)+ :atom_count atom-count+ :bucket_date composite-test-bucket-date+ :count cnt}))++(defn- cleanup-composite-rows! []+ (t2/delete! :model/SourceSegmentCompositeDaily :bucket_date composite-test-bucket-date))++(defn- composite-opts [source-id]+ {:source-type :table+ :source-id source-id+ :bucket-start composite-test-bucket-date+ :bucket-end composite-test-bucket-date})++(deftest suggested-segments-for-owner-happy-path-test+ (cleanup-composite-rows!)+ (memoize/memo-clear! composite-atomsets-memo)+ (let [fact (composite-fact-for-orders)+ opts (composite-opts (mt/id :orders))]+ (try+ (seed-composite-row! fact 3)+ (let [results (insights/suggested-segments-for-owner opts)]+ (testing "candidate is returned when no saved Segment matches"+ (is (seq results)))+ (testing "top candidate is a valid :and MBQL clause attributed to the right source"+ (let [{:keys [clause itemset-size source]} (first results)]+ (is (lib/clause-of-type? clause :and))+ (is (= (:atom-count fact) itemset-size))+ (is (= :table (:type source)))+ (is (= (mt/id :orders) (:id source))))))+ (finally+ (cleanup-composite-rows!)+ (memoize/memo-clear! composite-atomsets-memo)))))++(deftest suggested-segments-for-owner-skips-saved-segment-match-test+ (cleanup-composite-rows!)+ (memoize/memo-clear! composite-atomsets-memo)+ (let [fact (composite-fact-for-orders)+ opts (composite-opts (mt/id :orders))]+ (try+ (seed-composite-row! fact 3)+ (testing "precondition: candidate is present without a saved Segment"+ (is (seq (insights/suggested-segments-for-owner opts))))+ (memoize/memo-clear! composite-atomsets-memo)+ (mt/with-temp [:model/Segment _seg {:table_id (mt/id :orders)+ :definition (composite-orders-query)}]+ (let [results (insights/suggested-segments-for-owner opts)]+ (testing "candidate is filtered out when a saved Segment has the same atom-set"+ (is (not-any? (fn [{:keys [source itemset-size]}]+ (and (= :table (:type source))+ (= (mt/id :orders) (:id source))+ (= (:atom-count fact) itemset-size)))+ results)))))+ (finally+ (cleanup-composite-rows!)+ (memoize/memo-clear! composite-atomsets-memo)))))++(deftest existing-composite-atomsets-cached-test+ (testing "existing-composite-atomsets is TTL-memoized — repeated calls hit the DB once"+ (memoize/memo-clear! composite-atomsets-memo)+ (let [segment-selects (atom 0)+ real-select t2/select]+ (try+ (with-redefs [t2/select (fn [& args]+ (when (and (sequential? (first args))+ (= :model/Segment (ffirst args)))+ (swap! segment-selects inc))+ (apply real-select args))]+ (existing-composite-atomsets {:source-type :table :source-id (mt/id :orders)})+ (existing-composite-atomsets {:source-type :table :source-id (mt/id :orders)})+ (is (= 1 @segment-selects)))+ (finally+ (memoize/memo-clear! composite-atomsets-memo))))))++(deftest suggested-segments-for-owner-empty-when-no-rows-test+ (cleanup-composite-rows!)+ (memoize/memo-clear! composite-atomsets-memo)+ (try+ (is (= [] (insights/suggested-segments-for-owner+ (composite-opts (mt/id :orders)))))+ (finally+ (memoize/memo-clear! composite-atomsets-memo))))diff --git a/test/metabase/usage_metadata/store_test.clj b/test/metabase/usage_metadata/store_test.cljnew file mode 100644index 00000000000..197ba270d31--- /dev/null+++ b/test/metabase/usage_metadata/store_test.clj@@ -0,0 +1,93 @@+(ns metabase.usage-metadata.store-test+ (:require+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [metabase.usage-metadata.models.source-dimension-daily]+ [metabase.usage-metadata.models.source-dimension-profile-daily]+ [metabase.usage-metadata.models.source-metric-daily]+ [metabase.usage-metadata.models.source-segment-daily]+ [metabase.usage-metadata.settings :as usage-metadata.settings]+ [metabase.usage-metadata.store :as usage-metadata.store]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db))++(deftest replace-day!-test+ (let [bucket-date (t/local-date 2026 4 15)+ payload-a {:segments [{:source_type :table, :source_id 1, :ownership_mode :direct, :field_id 10, :predicate "pred-a", :bucket_date bucket-date, :count 2}+ {:source_type nil, :source_id nil, :ownership_mode :mixed, :field_id nil, :predicate "pred-mixed", :bucket_date bucket-date, :count 1}]+ :metrics [{:source_type :card, :source_id 2, :ownership_mode :direct, :agg_type :count, :agg_field_id nil, :temporal_field_id nil, :temporal_unit nil, :bucket_date bucket-date, :count 3}]+ :dimensions [{:source_type :table, :source_id 1, :ownership_mode :direct, :field_id 11, :temporal_unit :month, :binning nil, :bucket_date bucket-date, :count 4}]+ :profiles [{:source_type :table, :source_id 1, :field_id 11, :source_basis :fingerprint, :observation_type :single-value, :observation_value nil, :bucket_date bucket-date, :count 4}]}+ payload-b {:segments [{:source_type :table, :source_id 1, :ownership_mode :projected, :field_id 12, :predicate "pred-b", :bucket_date bucket-date, :count 5}]+ :metrics []+ :dimensions []+ :profiles []}]+ (try+ (usage-metadata.store/delete-day! bucket-date)+ (usage-metadata.store/replace-day! bucket-date payload-a)+ (is (= 2 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))+ (is (= 1 (t2/count :model/SourceMetricDaily :bucket_date bucket-date)))+ (is (= 1 (t2/count :model/SourceDimensionDaily :bucket_date bucket-date)))+ (is (= 1 (t2/count :model/SourceDimensionProfileDaily :bucket_date bucket-date)))+ (usage-metadata.store/replace-day! bucket-date payload-b)+ (is (= 1 (t2/count :model/SourceSegmentDaily :bucket_date bucket-date)))+ (is (= 0 (t2/count :model/SourceMetricDaily :bucket_date bucket-date)))+ (is (= 0 (t2/count :model/SourceDimensionDaily :bucket_date bucket-date)))+ (is (= 0 (t2/count :model/SourceDimensionProfileDaily :bucket_date bucket-date)))+ (is (= {:source_type :table, :ownership_mode :projected, :field_id 12, :predicate "pred-b", :count 5}+ (select-keys (t2/select-one :model/SourceSegmentDaily :bucket_date bucket-date)+ [:source_type :ownership_mode :field_id :predicate :count])))+ (finally+ (usage-metadata.store/delete-day! bucket-date)))))++(deftest replace-day!-does-not-affect-other-days-test+ (let [day-a (t/local-date 2026 4 15)+ day-b (t/local-date 2026 4 16)]+ (try+ (usage-metadata.store/delete-day! day-a)+ (usage-metadata.store/delete-day! day-b)+ (usage-metadata.store/replace-day! day-a {:segments [{:source_type :table, :source_id 1, :ownership_mode :direct, :field_id 10, :predicate "pred-a", :bucket_date day-a, :count 1}]+ :metrics []+ :dimensions []+ :profiles []})+ (usage-metadata.store/replace-day! day-b {:segments [{:source_type :table, :source_id 1, :ownership_mode :direct, :field_id 11, :predicate "pred-b", :bucket_date day-b, :count 1}]+ :metrics []+ :dimensions []+ :profiles []})+ (usage-metadata.store/replace-day! day-a {:segments [{:source_type :table, :source_id 1, :ownership_mode :direct, :field_id 12, :predicate "pred-c", :bucket_date day-a, :count 1}]+ :metrics []+ :dimensions []+ :profiles []})+ (is (= #{"pred-b"}+ (t2/select-fn-set :predicate :model/SourceSegmentDaily :bucket_date day-b)))+ (finally+ (usage-metadata.store/delete-day! day-a)+ (usage-metadata.store/delete-day! day-b)))))++(deftest replace-day!-transaction-rolls-back-on-failure-test+ (let [bucket-date (t/local-date 2026 4 15)+ original {:segments [{:source_type :table, :source_id 1, :ownership_mode :direct, :field_id 10, :predicate "pred-a", :bucket_date bucket-date, :count 2}]+ :metrics []+ :dimensions []+ :profiles []}]+ (try+ (usage-metadata.store/delete-day! bucket-date)+ (usage-metadata.store/replace-day! bucket-date original)+ (with-redefs [usage-metadata.store/insert-segment-rollups! (fn [_] (throw (ex-info "boom" {})))]+ (is (thrown? clojure.lang.ExceptionInfo+ (usage-metadata.store/replace-day! bucket-date original))))+ (is (= #{"pred-a"}+ (t2/select-fn-set :predicate :model/SourceSegmentDaily :bucket_date bucket-date)))+ (finally+ (usage-metadata.store/delete-day! bucket-date)))))++(deftest usage-metadata-settings-test+ (is (false? (usage-metadata.settings/usage-metadata-enabled?)))+ (is (= 90 (usage-metadata.settings/usage-metadata-retention-days)))+ (is (= "0 0 2 * * ? *" (usage-metadata.settings/usage-metadata-schedule)))+ (mt/with-temporary-setting-values [usage-metadata-last-completed-day "2026-04-14"]+ (is (= "2026-04-14"+ (usage-metadata.settings/usage-metadata-last-completed-day)))))diff --git a/test/metabase/usage_metadata/task/process_test.clj b/test/metabase/usage_metadata/task/process_test.cljnew file mode 100644index 00000000000..85294a11f37--- /dev/null+++ b/test/metabase/usage_metadata/task/process_test.clj@@ -0,0 +1,37 @@+(ns metabase.usage-metadata.task.process-test+ (:require+ [clojure.test :refer :all]+ [clojurewerkz.quartzite.jobs :as jobs]+ [metabase.task.core :as task]+ [metabase.test :as mt]+ [metabase.usage-metadata.batch]+ [metabase.usage-metadata.task.process :as usage-metadata.task.process]))++(set! *warn-on-reflection* true)++(defn- scheduled-job []+ (first (filter #(= "metabase.task.usage-metadata-process.job" (:key %))+ (mt/scheduler-current-tasks))))++(deftest init!-schedules-usage-metadata-job-test+ (mt/with-temp-scheduler!+ (mt/with-temporary-setting-values [usage-metadata-schedule "0 15 2 * * ? *"]+ (task/init! ::usage-metadata.task.process/UsageMetadataProcess)+ (let [job (scheduled-job)]+ (is (= "metabase.task.usage-metadata-process.job" (:key job)))+ (is (= "0 15 2 * * ? *"+ (get-in job [:triggers 0 :cron-schedule])))))))++(deftest usage-metadata-job-respects-enabled-setting-test+ (mt/with-temp-scheduler!+ (task/init! ::usage-metadata.task.process/UsageMetadataProcess)+ (let [ran? (promise)]+ (with-redefs [metabase.usage-metadata.batch/run-batch! (fn []+ (deliver ran? true))]+ (mt/with-temporary-setting-values [usage-metadata-enabled? false]+ (task/trigger-now! (jobs/key "metabase.task.usage-metadata-process.job"))+ (Thread/sleep 200)+ (is (= ::timeout (deref ran? 50 ::timeout))))+ (mt/with-temporary-setting-values [usage-metadata-enabled? true]+ (task/trigger-now! (jobs/key "metabase.task.usage-metadata-process.job"))+ (is (true? (deref ran? 5000 ::timeout))))))))