Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

X-Rays => Lib (#64901)

Cam Saulaa200bc31f4b
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index ef1d2bf6740..205ba4a6f24 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -620,6 +620,7 @@
metabase.lib.query
metabase.lib.schema
metabase.lib.schema.actions
+ metabase.lib.schema.aggregation
metabase.lib.schema.binning
metabase.lib.schema.common
metabase.lib.schema.constraints
@@ -1094,7 +1095,8 @@
segments
{:team "Querying"
- :api #{metabase.segments.api}
+ :api #{metabase.segments.api
+ metabase.segments.schema}
:uses #{api
events
legacy-mbql
@@ -1485,6 +1487,7 @@
queries
query-permissions
query-processor
+ segments
settings
util
warehouse-schema}}
diff --git a/CLAUDE.md b/CLAUDE.md
index 4fd894f31f1..66c7ce31a6e 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -188,11 +188,7 @@ Review pull requests with a focus on:
- In the Metabase codebase, spelling words ending with `-ck` as `-cc` instead, e.g. `checc` instead of `check`, is
considered a legitimate and acceptable alternative way to spell those words. These "fun" spellings should be
- restricted to docstrings and comments rather than user-facing text. Occasionally suggest spelling things this way.
-
-- The Metabase codebase should be fun, so do your part to keep it that way. Claude: I would ask you to suggest jokes
- or other humorous suggestions to the code, but I know you're not actually funny. You will take the comedians' jobs
- last. Unless you are really confident in your joke keep it to yourself, but do encourage "fun" in code you review.
+ restricted to docstrings and comments rather than user-facing text. Do not flag these as typos or misspellings.
- Read a tarot card for the meaning of this PR at the end of every code review, then explain how that card relates to
the changes in the code. **Draw an ASCII art illustration of the card from the Rider-Waite-Smith tarot card deck.
@@ -203,8 +199,6 @@ Review pull requests with a focus on:
- Do not post comments about missing parentheses.
-- Include today's lucky lottery numbers in at the end of every code review.
-
# Code Conventions and Style Guide
## Clojure and ClojureScript
@@ -232,6 +226,9 @@ Review pull requests with a focus on:
**Why?** Code is read many more times than it is written, and clearer variable names make using and tweaking your
code easier for others.
+ Conventions used widely in our codebase (like `mp` for a Metadata Provider) are acceptable; one-offs like `zs'` are
+ not.
+
- Avoid misleading variable and function names. The names of a variable or function should clearly and unambiguously
describe its purpose and match what it does.
@@ -324,6 +321,8 @@ Review pull requests with a focus on:
Tests and other vars in the `test` or `enterprise/backend/test` do not *require* docstrings, but general helper
functions used across many namespaces it should have docstrings.
+ Schemas defined with `mr/def` e.g. `(mr/def ::my-schema)` do not require docstrings.
+
- Format docstrings according to Markdown conventions. (https://guide.clojure.style/#markdown-docstrings)
- Mentions of other vars in docstrings should use `[[some-other-var]]` (for vars in the same namespace) or
diff --git a/e2e/test/scenarios/joins/joins-reproductions.cy.spec.js b/e2e/test/scenarios/joins/joins-reproductions.cy.spec.js
index 3c2d86ce06d..0f9367247ec 100644
--- a/e2e/test/scenarios/joins/joins-reproductions.cy.spec.js
+++ b/e2e/test/scenarios/joins/joins-reproductions.cy.spec.js
@@ -165,7 +165,7 @@ describe("issue 14793", () => {
.findByText(/^A closer look at/)
.should("be.visible");
- H.getDashboardCards().should("have.length", 18);
+ H.getDashboardCards().should("have.length", 35);
});
});
diff --git a/src/metabase/lib/aggregation.cljc b/src/metabase/lib/aggregation.cljc
index ce02411a2b7..47d36a45113 100644
--- a/src/metabase/lib/aggregation.cljc
+++ b/src/metabase/lib/aggregation.cljc
@@ -10,6 +10,7 @@
[metabase.lib.metadata.calculation :as lib.metadata.calculation]
[metabase.lib.options :as lib.options]
[metabase.lib.ref :as lib.ref]
+ [metabase.lib.remove-replace :as lib.remove-replace]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.aggregation :as lib.schema.aggregation]
[metabase.lib.schema.common :as lib.schema.common]
@@ -21,6 +22,7 @@
[metabase.util :as u]
[metabase.util.i18n :as i18n]
[metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
[metabase.util.performance :refer [select-keys mapv #?(:clj doseq) #?(:clj for)]]))
(mu/defn column-metadata->aggregation-ref :- :mbql.clause/aggregation
@@ -283,12 +285,12 @@
[aggregation-clause]
aggregation-clause)
-(def ^:private Aggregable
+(mr/def ::aggregable
"Schema for something you can pass to [[aggregate]] to add to a query as an aggregation."
[:or
- ::lib.schema.aggregation/aggregation
- ::lib.schema.common/external-op
- ::lib.schema.metadata/metric])
+ [:ref ::lib.schema.aggregation/aggregation]
+ [:ref ::lib.schema.common/external-op]
+ [:ref ::lib.schema.metadata/metric]])
(mu/defn aggregate :- ::lib.schema/query
"Adds an aggregation to query."
@@ -297,7 +299,7 @@
([query :- ::lib.schema/query
stage-number :- :int
- aggregable :- Aggregable]
+ aggregable :- ::aggregable]
;; if this is a Metric metadata, convert it to `:metric` MBQL clause before adding.
(if (= (lib.dispatch/dispatch-value aggregable) :metadata/metric)
(recur query stage-number (lib.ref/ref aggregable))
@@ -481,3 +483,16 @@
[query stage-number ag-ref]
(let [expression (resolve-aggregation query stage-number ag-ref)]
(lib.metadata.calculation/type-of query stage-number expression)))
+
+(mu/defn remove-all-aggregations :- ::lib.schema/query
+ "Remove all aggregations from a query stage."
+ ([query]
+ (remove-all-aggregations query -1))
+
+ ([query :- ::lib.schema/query
+ stage-number :- :int]
+ (reduce
+ (fn [query ag]
+ (lib.remove-replace/remove-clause query stage-number ag))
+ query
+ (aggregations query stage-number))))
diff --git a/src/metabase/lib/common.cljc b/src/metabase/lib/common.cljc
index c20b52a0676..300a1de6354 100644
--- a/src/metabase/lib/common.cljc
+++ b/src/metabase/lib/common.cljc
@@ -61,9 +61,10 @@
(defmethod ->op-arg :lib/external-op
[{:keys [operator options args] :or {options {}}}]
- (->op-arg (lib.options/ensure-uuid (into [(keyword operator) options]
- (map ->op-arg)
- args))))
+ (->op-arg (-> (lib.options/ensure-uuid (into [(keyword operator) options]
+ (map ->op-arg)
+ args))
+ ((#?(:clj requiring-resolve :cljs resolve) 'metabase.lib.normalize/normalize)))))
(defn defop-create
"Impl for [[defop]]."
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index 83ff2ab4e58..db96841c42b 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -127,6 +127,7 @@
aggregations
aggregations-metadata
available-aggregation-operators
+ remove-all-aggregations
selected-aggregation-operators
count
avg
diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljc
index 5d8a7b0bd64..3563ba4306b 100644
--- a/src/metabase/lib/schema.cljc
+++ b/src/metabase/lib/schema.cljc
@@ -143,18 +143,26 @@
acc))
stage stage))
+(def ^:dynamic *HACK-disable-ref-validation*
+ "Whether to validate join aliases in field refs and expression refs. This is only disable-able as a hack to support
+ X-Rays code which generates fragments of stages that drop joins and expressions and then adds them again after the
+ fact in [[metabase.xrays.automagic-dashboards.core/preserve-entity-element]]. Once we port X-Rays to use Lib we can
+ fix the hackiness and hopefully take this out."
+ false)
+
(defn- expression-ref-errors-for-stage [stage]
- (let [stage (dissoc stage :parameters) ; don't validate [:dimension [:expression ...]] refs since they might not be moved to the correct place yet.
- expression-names (when-let [expressions (:expressions stage)]
- (when (and (sequential? expressions)
- (every? sequential? expressions))
- (into #{} (map (comp :lib/expression-name second)) expressions)))
- pred #(bad-ref-clause? :expression expression-names %)
- form (-> (stage-with-joins-and-namespaced-keys-removed stage)
- ;; also ignore expression refs inside `:parameters` since they still use legacy syntax these days.
- (dissoc :parameters))]
- (when (lib.schema.util/pred-matches-form? form pred)
- (lib.schema.util/matching-locations form pred))))
+ (when-not *HACK-disable-ref-validation*
+ (let [stage (dissoc stage :parameters) ; don't validate [:dimension [:expression ...]] refs since they might not be moved to the correct place yet.
+ expression-names (when-let [expressions (:expressions stage)]
+ (when (and (sequential? expressions)
+ (every? sequential? expressions))
+ (into #{} (map (comp :lib/expression-name second)) expressions)))
+ pred #(bad-ref-clause? :expression expression-names %)
+ form (-> (stage-with-joins-and-namespaced-keys-removed stage)
+ ;; also ignore expression refs inside `:parameters` since they still use legacy syntax these days.
+ (dissoc :parameters))]
+ (when (lib.schema.util/pred-matches-form? form pred)
+ (lib.schema.util/matching-locations form pred)))))
(defn- aggregation-ref-errors-for-stage [stage]
(let [uuids (into #{} (map (comp :lib/uuid second)) (:aggregation stage))
@@ -348,17 +356,10 @@
(mapcat join-aliases-in-join (:joins stage)))]
(set (join-aliases-in-stage stage)))))
-(def ^:dynamic *HACK-disable-join-alias-in-field-ref-validation*
- "Whether to validate join aliases in field refs. This is only disable-able as a hack to support X-Rays code which
- generates fragments of stages that drop joins and then adds them again after the fact
- in [[metabase.xrays.automagic-dashboards.core/preserve-entity-element]]. Once we port X-Rays to use Lib we can fix
- the hackiness and hopefully take this out."
- false)
-
(defn- join-ref-error-for-stages
"Return an error messages if we find a field ref that uses a `:join-alias` for a join that doesn't exist."
[stages]
- (when (and (not *HACK-disable-join-alias-in-field-ref-validation*)
+ (when (and (not *HACK-disable-ref-validation*)
(sequential? stages))
(loop [visible-join-alias? (constantly false), i 0, [stage & more] stages]
(let [visible-join-alias? (some-fn visible-join-alias? (visible-join-alias?-fn stage))]
diff --git a/src/metabase/lib/schema/common.cljc b/src/metabase/lib/schema/common.cljc
index aae19f9d3c6..df27ee626af 100644
--- a/src/metabase/lib/schema/common.cljc
+++ b/src/metabase/lib/schema/common.cljc
@@ -287,8 +287,8 @@
[:operator [:multi {:dispatch string?}
[true :string]
[false :keyword]]]
- [:args [:sequential :any]]
- [:options {:optional true} ::options]])
+ [:args [:schema {:decode/normalize vec} [:sequential :any]]]
+ [:options {:optional true} ::options]])
#?(:clj
(defn- instance-of-class* [& classes]
diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.clj
index 26ecf0756a1..8bc93fe3fde 100644
--- a/src/metabase/lib_be/metadata/jvm.clj
+++ b/src/metabase/lib_be/metadata/jvm.clj
@@ -45,10 +45,11 @@
(def ^:private metadata-type->schema
{:metadata/card ::lib.schema.metadata/card})
-(defn instance->metadata
+(mu/defn instance->metadata
"Convert a (presumably) Toucan 2 instance of an application database model with `snake_case` keys to a MLv2 style
metadata instance with `:lib/type` and `kebab-case` keys."
- [instance metadata-type]
+ [instance :- :map
+ metadata-type :- :keyword]
(let [normalize (if-let [schema (get metadata-type->schema metadata-type)]
(fn [instance]
(lib.normalize/normalize schema instance))
diff --git a/src/metabase/query_processor/store.clj b/src/metabase/query_processor/store.clj
index b058becfe68..b934f8dd3a5 100644
--- a/src/metabase/query_processor/store.clj
+++ b/src/metabase/query_processor/store.clj
@@ -216,7 +216,7 @@
;;;; DEPRECATED STUFF
;;;;
-(defn ->legacy-metadata
+(mu/defn ->legacy-metadata
"For compatibility: convert MLv2-style metadata as returned by [[metabase.lib.metadata.protocols]]
or [[metabase.lib.metadata]] functions
(with `kebab-case` keys and `:lib/type`) to legacy QP/application database style metadata (with `snake_case` keys
@@ -227,11 +227,8 @@
(Note: it is preferable to use [[metabase.lib.core/lib-metadata-column->legacy-metadata-column]] instead of this
function if you REALLY need to do this sort of conversion.)"
{:deprecated "0.48.0"}
- [lib-metadata-col]
- (let [model (case (:lib/type lib-metadata-col)
- :metadata/database :model/Database
- :metadata/table :model/Table
- :metadata/column :model/Field)]
- (-> lib-metadata-col
- lib/lib-metadata-column->legacy-metadata-column
- (vary-meta assoc :type model))))
+ [lib-metadata-col :- [:map
+ [:lib/type [:= :metadata/column]]]]
+ (-> lib-metadata-col
+ lib/lib-metadata-column->legacy-metadata-column
+ (vary-meta assoc :type :metadata/column)))
diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.clj
index b5a4753edfd..17e85e549c5 100644
--- a/src/metabase/segments/models/segment.clj
+++ b/src/metabase/segments/models/segment.clj
@@ -7,7 +7,6 @@
[metabase.api.common :as api]
;; existing usages, do not use legacy MBQL utils in new code
^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]
^{:clj-kondo/ignore [:discouraged-namespace :deprecated-namespace]} [metabase.legacy-mbql.util :as mbql.u]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
@@ -20,6 +19,7 @@
[metabase.models.serialization :as serdes]
[metabase.permissions.core :as perms]
[metabase.search.core :as search]
+ [metabase.segments.schema :as segments.schema]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
@@ -33,14 +33,9 @@
(methodical/defmethod t2/table-name :model/Segment [_model] :segment)
(methodical/defmethod t2/model-for-automagic-hydration [:default :segment] [_original-model _k] :model/Segment)
-(mr/def ::segment-definition
- [:map
- [:filter {:optional true} [:maybe mbql.s/Filter]]
- [:aggregation {:optional true} [:maybe [:sequential ::mbql.s/Aggregation]]]])
-
(defn- validate-segment-definition
[definition]
- (if-let [error (mr/explain ::segment-definition definition)]
+ (if-let [error (mr/explain ::segments.schema/segment definition)]
(let [humanized (me/humanize error)]
(throw (ex-info (tru "Invalid Metric or Segment: {0}" (pr-str humanized))
{:error error
diff --git a/src/metabase/segments/schema.clj b/src/metabase/segments/schema.clj
new file mode 100644
index 00000000000..faaeda03f10
--- /dev/null
+++ b/src/metabase/segments/schema.clj
@@ -0,0 +1,10 @@
+(ns metabase.segments.schema
+ (:require
+ ;; existing usage, do not use legacy MBQL utils in new code
+ ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]
+ [metabase.util.malli.registry :as mr]))
+
+(mr/def ::segment
+ [:map
+ [:filter {:optional true} [:maybe mbql.s/Filter]]
+ [:aggregation {:optional true} [:maybe [:sequential ::mbql.s/Aggregation]]]])
diff --git a/src/metabase/xrays/api/automagic_dashboards.clj b/src/metabase/xrays/api/automagic_dashboards.clj
index 2d2cb3e5259..b2102b19484 100644
--- a/src/metabase/xrays/api/automagic_dashboards.clj
+++ b/src/metabase/xrays/api/automagic_dashboards.clj
@@ -5,6 +5,8 @@
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.models.interface :as mi]
[metabase.queries.core :as queries]
[metabase.query-permissions.core :as query-perms]
@@ -263,11 +265,12 @@
:keys [linked-tables]}]
(if (seq linked-tables)
(let [child-dashboards (map (fn [{:keys [linked-table-id linked-field-id]}]
- (let [table (t2/select-one :model/Table :id linked-table-id)]
+ (let [table (t2/select-one :model/Table :id linked-table-id)
+ mp (lib-be/application-database-metadata-provider (:db_id table))]
(automagic-dashboards.core/automagic-analysis
table
{:show :all
- :query-filter [:= [:field linked-field-id nil] model_pk]})))
+ :query-filter [(lib/= (lib.metadata/field mp linked-field-id) model_pk)]})))
linked-tables)
seed-dashboard (-> (first child-dashboards)
(merge
diff --git a/src/metabase/xrays/automagic_dashboards/combination.clj b/src/metabase/xrays/automagic_dashboards/combination.clj
index 7a92eba6d3d..d8673ae110f 100644
--- a/src/metabase/xrays/automagic_dashboards/combination.clj
+++ b/src/metabase/xrays/automagic_dashboards/combination.clj
@@ -21,12 +21,12 @@
[clojure.walk :as walk]
[medley.core :as m]
[metabase.driver.util :as driver.u]
- ;; legacy usage, do not use this in new code
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]
- [metabase.lib.schema.id :as lib.schema.id]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.models.interface :as mi]
[metabase.queries.core :as queries]
- [metabase.query-processor.util :as qp.util]
[metabase.util :as u]
[metabase.util.i18n :as i18n]
[metabase.util.malli :as mu]
@@ -36,19 +36,20 @@
[metabase.xrays.automagic-dashboards.util :as magic.util]
[metabase.xrays.automagic-dashboards.visualization-macros :as visualization]))
-(defn add-breakouts-and-filter
+(mu/defn add-breakouts-and-filters :- ::ads/grounded-metric.definition
"Add breakouts and filters to a query based on the breakout fields and filter clauses"
- [query
- breakout-fields
- filter-clauses]
- (let [breakouts (mapv (partial interesting/->reference :mbql) breakout-fields)]
- (cond-> (assoc query :breakout breakouts)
- (seq filter-clauses) (assoc :filter (into [:and] filter-clauses)))))
-
-(defn- add-aggregations
+ [metric-def :- ::ads/grounded-metric.definition
+ breakouts :- [:maybe [:sequential ::lib.schema.metadata/column]]
+ filter-clauses :- [:maybe [:sequential ::ads/external-op]]]
+ (assoc metric-def
+ :xrays/breakouts breakouts
+ :xrays/filters filter-clauses))
+
+(mu/defn- add-aggregations :- ::ads/grounded-metric.definition
"Add aggregations to a query."
- [query aggregations]
- (assoc query :aggregation aggregations))
+ [metric-def :- ::ads/grounded-metric.definition
+ aggregations :- [:sequential ::ads/aggregation]]
+ (assoc metric-def :xrays/aggregations aggregations))
(defn matching-types?
"Given two seqs of types, return true of the types of the child
@@ -82,21 +83,29 @@
[:dataset_query ::ads/query]]
"Add the `:dataset_query` key to this metric. Requires both the current metric-definition (from the grounded metric)
and the database and table ids (from the source object)."
- [{:keys [metric-definition] :as ground-metric-with-dimensions}
+ [{:keys [metric-definition] :as ground-metric-with-dimensions} :- ::ads/grounded-metric
{{:keys [database]} :root :keys [source query-filter]} :- ::ads/context]
- (let [source-table (if (->> source (mi/instance-of? :model/Table))
- (-> source u/the-id)
- (->> source u/the-id (str "card__")))
- model? (and (mi/instance-of? :model/Card source)
- (queries/model? source))]
- (assoc ground-metric-with-dimensions
- :dataset_query (-> {:database database
- :type :query
- :query (cond-> (assoc metric-definition
- :source-table source-table)
- (and (not model?)
- query-filter) (assoc :filter query-filter))}
- mbql.normalize/normalize))))
+ (let [model? (and (mi/instance-of? :model/Card source)
+ (queries/model? source))
+ mp (lib-be/application-database-metadata-provider database)
+ query (cond-> (lib/query mp (cond
+ (mi/instance-of? :model/Table source) (lib.metadata/table mp (u/the-id source))
+ (mi/instance-of? :model/Card source) (lib.metadata/card mp (u/the-id source))))
+ (:xrays/aggregations metric-definition)
+ (as-> $query (reduce lib/aggregate $query (:xrays/aggregations metric-definition)))
+
+ (:xrays/breakouts metric-definition)
+ (as-> $query (reduce lib/breakout $query (:xrays/breakouts metric-definition)))
+
+ (:xrays/filters metric-definition)
+ (as-> $query (reduce lib/filter $query (:xrays/filters metric-definition)))
+
+ (not model?)
+ (as-> $query (reduce
+ lib/filter
+ $query
+ query-filter)))]
+ (assoc ground-metric-with-dimensions :dataset_query query)))
(defn- instantiate-visualization
[[k v] dimensions metrics]
@@ -125,7 +134,7 @@
(str/replace s #"\[\[(\w+)(?:\.([\w\-]+))?\]\]"
(fn [[_ identifier attribute]]
(let [entity (binding-fn identifier)
- attribute (some-> attribute qp.util/normalize-token)]
+ attribute (some-> attribute u/->kebab-case-en keyword)]
(str (or (and (ifn? entity) (entity attribute))
(root attribute)
(interesting/->reference template-type entity))))))))
@@ -156,22 +165,6 @@
dimension-name->field
(map first card-dimensions)))
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(def ^:private ^{:arglists '([field])} id-or-name
- (some-fn :id :name))
-
-(defn- singular-cell-dimensions
- [{:keys [cell-query]}]
- (letfn [(collect-dimensions [[op & args]]
- (case (some-> op qp.util/normalize-token)
- :and (mapcat collect-dimensions args)
- (:between :=) (magic.util/collect-field-references args)
- nil))]
- (->> cell-query
- collect-dimensions
- (map magic.util/field-reference->id)
- set)))
-
(defn- valid-breakout-dimension?
[{:keys [base_type db fingerprint aggregation]}]
(or (nil? aggregation)
@@ -181,30 +174,25 @@
(not= (-> fingerprint :type :type/Number :min)
(-> fingerprint :type :type/Number :max)))))
-(defn- valid-bindings? [{:keys [root]} satisfied-dimensions bindings]
- (let [cell-dimension? (singular-cell-dimensions root)]
- (->> satisfied-dimensions
- (map first)
- (map (fn [[identifier opts]]
- (merge (bindings identifier) opts)))
- (every? (every-pred valid-breakout-dimension?
- (complement (comp cell-dimension? id-or-name)))))))
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+(defn- valid-bindings? [satisfied-dimensions bindings]
+ (->> satisfied-dimensions
+ (map first)
+ (map (fn [[identifier opts]]
+ (merge (bindings identifier) opts)))
+ (every? valid-breakout-dimension?)))
(mu/defn grounded-metrics->dashcards :- [:sequential
[:merge
- ads/combined-metric
+ ::ads/combined-metric
[:map
- [:dataset_query
- [:map
- [:database ::lib.schema.id/database]]]]]]
+ [:dataset_query ::ads/query]]]]
"Generate dashcards from ground dimensions, using the base context, ground dimensions,
card templates, and grounded metrics as input."
[base-context :- ::ads/context
card-templates
- ground-dimensions :- ads/dim-name->matching-fields
- ground-filters
- grounded-metrics :- [:sequential ads/grounded-metric]]
+ ground-dimensions :- ::ads/dim-name->matching-fields
+ ground-filters :- [:sequential ::ads/grounded-filter]
+ grounded-metrics :- [:sequential ::ads/grounded-metric]]
(let [metric-name->metric (zipmap
(map :metric-name grounded-metrics)
(map-indexed
@@ -226,12 +214,11 @@
(apply math.combo/cartesian-product)
(map (partial zipmap dim-names)))
:let [merged-dims (combine-dimensions dimension-name->field card-dimensions)]
- :when (and (valid-bindings? base-context card-dimensions dimension-name->field)
+ :when (and (valid-bindings? card-dimensions dimension-name->field)
(every? metric-name->metric card-metrics))
:let [[grounded-metric :as all-satisfied-metrics] (map metric-name->metric card-metrics)
final-aggregate (into []
- (comp (map (comp :aggregation :metric-definition))
- cat)
+ (mapcat (comp :xrays/aggregations :metric-definition))
all-satisfied-metrics)
bound-metric-dimension-name->field (apply merge (map :dimension-name->field all-satisfied-metrics))
all-names->field (into dimension-name->field bound-metric-dimension-name->field)
@@ -249,15 +236,18 @@
card
(-> grounded-metric
(assoc
- :id (gensym)
+ :id (gensym)
:affinity-name card-name
- :card-score card-score
- :total-score (long (/ (apply + score-components) (count score-components)))
+ :card-score card-score
+ :total-score (long (/ (apply + score-components) (count score-components)))
;; Update dimension-name->field to include named contributions from both metrics and dimensions
:dimension-name->field all-names->field
- :score-components score-components)
- (update :metric-definition add-aggregations final-aggregate)
- (update :metric-definition add-breakouts-and-filter
- (vals merged-dims)
- (mapv (comp :filter simple-grounded-filters) card-filters))
+ :score-components score-components)
+ (update :metric-definition (fn [metric-definition]
+ (-> metric-definition
+ (add-aggregations final-aggregate)
+ (add-breakouts-and-filters (for [field (vals merged-dims)]
+ (interesting/field->metadata field))
+ (mapv (comp :filter simple-grounded-filters)
+ card-filters)))))
(add-dataset-query base-context))))))
diff --git a/src/metabase/xrays/automagic_dashboards/comparison.clj b/src/metabase/xrays/automagic_dashboards/comparison.clj
index 519db963352..08bd000a048 100644
--- a/src/metabase/xrays/automagic_dashboards/comparison.clj
+++ b/src/metabase/xrays/automagic_dashboards/comparison.clj
@@ -2,8 +2,9 @@
(:require
[medley.core :as m]
[metabase.api.common :as api]
- ;; legacy usage, do not use this in new code
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]
+ [metabase.lib.core :as lib]
+ [metabase.lib.equality :as lib.equality]
+ [metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.models.interface :as mi]
[metabase.util :as u]
@@ -53,36 +54,30 @@
instead, but that validates its input and output, and the queries that come in here aren't always valid (for
example, missing `:database`). If we can, it would be nice to use that instead of reinventing the wheel here."
[query :- ::ads/query
- new-filter-clauses :- [:maybe [:sequential :any]]]
- (magic.util/do-with-legacy-query
- query
- (fn [{{existing-filter-clause :filter} :query, :as query}]
- (let [clauses (filter identity (cons existing-filter-clause new-filter-clauses))
- new-filter-clause (when (seq clauses)
- ;; existing usage, this namespace will be rewritten to use Lib soon.
- #_{:clj-kondo/ignore [:deprecated-var]}
- (mbql.normalize/normalize-fragment [:query :filter] (cons :and clauses)))]
- (cond-> query
- (seq new-filter-clause) (magic.util/do-with-legacy-query assoc-in [:query :filter] new-filter-clause))))))
+ new-filter-clauses :- [:maybe [:sequential [:maybe ::ads/filter-clause]]]]
+ (reduce (fn [query filter-clause]
+ (cond-> query
+ filter-clause
+ (lib/filter filter-clause)))
+ query
+ new-filter-clauses))
-(defn- inject-filter
+(mu/defn- inject-filter
"Inject filter clause into card."
- [{:keys [query-filter cell-query] :as root} card]
+ [{:keys [query-filter cell-query] :as root} :- ::ads/root
+ card :- [:map
+ [:dataset_query ::ads/query]]]
(-> card
- (update :dataset_query #(add-filter-clauses % [query-filter cell-query]))
+ (update :dataset_query #(add-filter-clauses % (cons cell-query query-filter)))
(update :series (partial map (partial inject-filter root)))))
(mu/defn- multiseries?
[card :- [:map
[:dataset_query {:optional true} ::ads/query]]]
(or (-> card :series not-empty)
- (some-> card
- :dataset_query
- not-empty
- (magic.util/do-with-legacy-query
- (fn [query]
- (or (-> query (get-in [:query :aggregation]) count (> 1))
- (-> query (get-in [:query :breakout]) count (> 1))))))))
+ (when-let [query (not-empty (:dataset_query card))]
+ (or (-> query lib/aggregations count (> 1))
+ (-> query lib/breakouts count (> 1))))))
(mu/defn- overlay-comparison?
[card :- [:map
@@ -97,7 +92,7 @@
card-left (->> card (inject-filter left) clone-card)
card-right (->> card (inject-filter right) clone-card)
[color-left color-right] (->> [left right]
- (map #(magic.util/do-with-legacy-query (:dataset_query %) get-in [:query :filter]))
+ (map #(some-> % :dataset_query lib/filters))
populate/map-to-colors)]
(if (overlay-comparison? card)
(let [card (-> card-left
@@ -185,7 +180,7 @@
[:dataset_query {:optional true} ::ads/query]]]
(get-in card [:visualization_settings :graph.series_labels]
(map (comp capitalize-first names/metric-name)
- (magic.util/do-with-legacy-query (:dataset_query card) get-in [:query :aggregation]))))
+ (lib/aggregations (:dataset_query card)))))
(mu/defn- unroll-multiseries
[card :- [:map
@@ -193,10 +188,13 @@
(if (and (multiseries? card)
(-> card :display (= :line)))
(for [[aggregation label] (map vector
- (magic.util/do-with-legacy-query (:dataset_query card) get-in [:query :aggregation])
+ (lib/aggregations (:dataset_query card))
(series-labels card))]
(-> card
- (update :dataset_query magic.util/do-with-legacy-query assoc-in [:query :aggregation] [aggregation])
+ (update :dataset_query (fn [query]
+ (-> query
+ lib/remove-all-aggregations
+ (lib/aggregate aggregation))))
(assoc :name label)
(m/dissoc-in [:visualization_settings :graph.series_labels])))
[card]))
@@ -248,67 +246,68 @@
(and ((some-fn :cell-query :query-filter) left)
(not ((some-fn :cell-query :query-filter) right))))
-(defn comparison-dashboard
+(mu/defn comparison-dashboard
"Create a comparison dashboard based on dashboard `dashboard` comparing subsets of
the dataset defined by segments `left` and `right`."
[dashboard left right opts]
- (let [left (-> left
- ->root
- (merge (:left opts)))
- right (-> right
- ->root
- (merge (:right opts)))
- left (cond-> left
- (-> opts :left :cell-query)
- (assoc :comparison-name (->> opts
- :left
- :cell-query
- (names/cell-title left))))
- right (cond-> right
- (part-vs-whole-comparison? left right)
- (assoc :comparison-name (condp mi/instance-of? (:entity right)
- :model/Table
- (tru "All {0}" (:short-name right))
-
- (tru "{0}, all {1}"
- (comparison-name right)
- (names/source-name right)))))
- segment-dashboards (->> (concat (segment-constituents left)
- (segment-constituents right))
- distinct
- (map #(automagic-analysis % {:source (:source left)
- :rules-prefix ["comparison"]})))]
- (assert (or (let [left-source (m/update-existing (:source left) :dataset_query magic.util/do-with-legacy-query identity)
- right-source (m/update-existing (:source right) :dataset_query magic.util/do-with-legacy-query identity)]
- (= left-source right-source))
- (= (-> left :source :table_id)
- (-> right :source u/the-id))))
- (->> (concat segment-dashboards [dashboard])
- (reduce (fn [dashboard-1 dashboard-2]
- (if dashboard-1
- (populate/merge-dashboards dashboard-1 dashboard-2 {:skip-titles? true})
- dashboard-2))
- nil)
- dashboard->cards
- (m/distinct-by (some-fn :dataset_query hash))
- (transduce (mapcat unroll-multiseries)
- (fn
- ([]
- (let [title (tru "Comparison of {0} and {1}"
- (comparison-name left)
- (comparison-name right))]
- (-> {:name title
- :transient_name title
- :transient_filters nil
- :param_fields nil
- :description (tru "Automatically generated comparison dashboard comparing {0} and {1}"
- (comparison-name left)
- (comparison-name right))
- :creator_id api/*current-user-id*
- :parameters []
- :related (update-related (:related dashboard) left right)}
- (add-title-row left right))))
- ([[dashboard _row]] dashboard)
- ([[dashboard row] card]
- [(comparison-row dashboard row left right card)
- (+ row (:height card))]))))))
+ (binding [lib.schema/*HACK-disable-ref-validation* true]
+ (let [opts (-> opts
+ (m/update-existing-in [:left :cell-query] (partial lib/normalize ::ads/root.cell-query))
+ (m/update-existing-in [:right :cell-query] (partial lib/normalize ::ads/root.cell-query)))
+ left (-> left
+ ->root
+ (merge (:left opts)))
+ right (-> right
+ ->root
+ (merge (:right opts)))
+ left (cond-> left
+ (-> opts :left :cell-query)
+ (assoc :comparison-name (->> opts
+ :left
+ :cell-query
+ (names/cell-title left))))
+ right (cond-> right
+ (part-vs-whole-comparison? left right)
+ (assoc :comparison-name (if (mi/instance-of? :model/Table (:entity right))
+ (tru "All {0}" (:short-name right))
+ (tru "{0}, all {1}"
+ (comparison-name right)
+ (names/source-name right)))))
+ segment-dashboards (->> (concat (segment-constituents left)
+ (segment-constituents right))
+ distinct
+ (map #(automagic-analysis % {:source (:source left)
+ :rules-prefix ["comparison"]})))]
+ (assert (or (lib.equality/= (get-in left [:source :dataset_query])
+ (get-in right [:source :dataset_query]))
+ (= (-> left :source :table_id)
+ (-> right :source u/the-id))))
+ (->> (concat segment-dashboards [dashboard])
+ (reduce (fn [dashboard-1 dashboard-2]
+ (if dashboard-1
+ (populate/merge-dashboards dashboard-1 dashboard-2 {:skip-titles? true})
+ dashboard-2))
+ nil)
+ dashboard->cards
+ (m/distinct-by (some-fn :dataset_query hash))
+ (transduce (mapcat unroll-multiseries)
+ (fn
+ ([]
+ (let [title (tru "Comparison of {0} and {1}"
+ (comparison-name left)
+ (comparison-name right))]
+ (-> {:name title
+ :transient_name title
+ :transient_filters nil
+ :param_fields nil
+ :description (tru "Automatically generated comparison dashboard comparing {0} and {1}"
+ (comparison-name left)
+ (comparison-name right))
+ :creator_id api/*current-user-id*
+ :parameters []
+ :related (update-related (:related dashboard) left right)}
+ (add-title-row left right))))
+ ([[dashboard _row]] dashboard)
+ ([[dashboard row] card]
+ [(comparison-row dashboard row left right card)
+ (+ row (:height card))])))))))
diff --git a/src/metabase/xrays/automagic_dashboards/core.clj b/src/metabase/xrays/automagic_dashboards/core.clj
index 68039936194..0808f4048a7 100644
--- a/src/metabase/xrays/automagic_dashboards/core.clj
+++ b/src/metabase/xrays/automagic_dashboards/core.clj
@@ -64,12 +64,12 @@
## The Dynamic Binding and Dashboard Generation Process ##
- Once data has been accreted in `automagic-analysis`, `automagic-dashboard` will first select a template as described
- above. It then calls `metabase.xrays.automagic-dashboards.interesting/identify` which takes the actual column data and
+ Once data has been accreted in [[automagic-analysis]], [[automagic-dashboard]] will first select a template as described
+ above. It then calls [[metabase.xrays.automagic-dashboards.interesting/identify]] which takes the actual column data and
dimension, metric, and filter definitions from the template and matches all potential columns to potential dimensions,
- metrics, and filters. The resulting \"grounded-values\" are now passed into `generate-dashboard`, which matches all
+ metrics, and filters. The resulting \"grounded-values\" are now passed into [[generate-dashboard]], which matches all
of these values to card templates to produce a dashboard. The majority of the card generation work is done in
- `metabase.xrays.automagic-dashboards.combination/grounded-metrics->dashcards`.
+ [[metabase.xrays.automagic-dashboards.combination/grounded-metrics->dashcards]].
Note that if a card template's dimensions, metrics, and filters are not matched to grounded values the card will not
be generated. Conversely, if a card template can be matched by multiple combinations of dimensions, multiple cards
@@ -148,17 +148,18 @@
[kixi.stats.math :as math]
[medley.core :as m]
[metabase.analyze.core :as analyze]
- ;; legacy usage, do not use this in new code
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]
[metabase.lib.core :as lib]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.lib.schema.aggregation :as lib.schema.aggregation]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.models.interface :as mi]
[metabase.query-processor.util :as qp.util]
+ [metabase.segments.schema :as segments.schema]
[metabase.util :as u]
[metabase.util.i18n :as i18n :refer [tru trun]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[metabase.warehouse-schema.models.field :as field]
[metabase.xrays.automagic-dashboards.combination :as combination]
@@ -195,7 +196,7 @@
:dashboard-templates-prefix ["table"]})
(mu/defmethod ->root :model/Segment :- ::ads/root
- [segment]
+ [segment :- ::segments.schema/segment]
(let [table (->> segment :table_id (t2/select-one :model/Table :id))]
{:entity segment
:full-name (tru "{0} in the {1} segment" (:display_name table) (:name segment))
@@ -203,13 +204,13 @@
:comparison-name (tru "{0} segment" (:name segment))
:source table
:database (:db_id table)
- :query-filter [:segment (u/the-id segment)]
+ :query-filter [(lib/segment (u/the-id segment))]
:url (format "%ssegment/%s" public-endpoint (u/the-id segment))
:dashboard-templates-prefix ["table"]}))
(mu/defmethod ->root :xrays/Metric :- ::ads/root
- [metric :- ::ads/metric]
- (let [table (some->> metric :table_id (t2/select-one :model/Table :id))]
+ [{:keys [table-id], :as metric} :- ::ads/metric]
+ (let [table (some->> table-id (t2/select-one :model/Table :id))]
{:entity metric
:full-name (if (:id metric)
(trun "{0} metric" "{0} metrics" (:name metric))
@@ -238,7 +239,7 @@
(mu/defn- source-card-id [card-or-question :- [:map
[:dataset_query ::ads/query]]]
- (magic.util/do-with-mbql5-query (:dataset_query card-or-question) lib/source-card-id))
+ (lib/source-card-id (:dataset_query card-or-question)))
(mu/defn- nested-query?
"Is this card or question derived from another model or question?"
@@ -250,7 +251,7 @@
"Is this card or question native (SQL)?"
[{query :dataset_query, :as _card-or-question} :- [:map
[:dataset_query ::ads/query]]]
- (magic.util/do-with-legacy-query query #(= (:type %) :native)))
+ (lib/native-only-query? query))
(mu/defn- source-question :- [:maybe (ms/InstanceOf :model/Card)]
[card-or-question :- [:map
@@ -259,14 +260,11 @@
(t2/select-one :model/Card :id source-card-id)))
(mu/defn- table-like?
- [card-or-question :- [:map
- [:dataset_query ::ads/query]]]
- (magic.util/do-with-legacy-query
- (:dataset_query card-or-question)
- (fn [query]
- (and
- (nil? (get-in query [:query :aggregation]))
- (nil? (get-in query [:query :breakout]))))))
+ [{query :dataset_query, :as _card-or-question} :- [:map
+ [:dataset_query ::ads/query]]]
+ (and
+ (empty? (lib/aggregations query))
+ (empty? (lib/breakouts query))))
(defn- table-id
"Get the Table ID from `card-or-question`, which can be either a Card from the DB (which has a `:table_id` property)
@@ -275,8 +273,7 @@
;; TODO - probably better if we just changed `adhoc-query` to use the same keys as Cards (e.g. `:table_id`) so we
;; didn't need this function, seems like something that would be too easy to forget
[card-or-question]
- (or (:table_id card-or-question)
- (:table-id card-or-question)))
+ ((some-fn :table-id :table_id) card-or-question))
(mu/defn- source
[card :- [:map
@@ -293,17 +290,13 @@
:else (->> card table-id (t2/select-one :model/Table :id))))
(mu/defmethod ->root :model/Card :- ::ads/root
- [card]
- (let [card (update card
- :dataset_query
- ;; temporary until we rework X-rays to use Lib/MBQL 5
- #_{:clj-kondo/ignore [:discouraged-var]}
- lib/->legacy-MBQL)
- source (source card)]
+ [card :- [:map
+ [:dataset_query ::ads/query]]]
+ (let [source (source card)]
{:entity card
:source source
:database (:database_id card)
- :query-filter (magic.util/do-with-legacy-query (:dataset_query card) get-in [:query :filter])
+ :query-filter (lib/filters (:dataset_query card))
:full-name (tru "\"{0}\"" (:name card))
:short-name (names/source-name {:source source})
:url (format "%s%s/%s" public-endpoint (name (:type source :question)) (u/the-id card))
@@ -314,14 +307,12 @@
(mu/defmethod ->root :model/Query :- ::ads/root
[query :- [:map
[:database-id ::lib.schema.id/database]]]
- ;; use of [[lib/->legacy-MBQL]] here is temporary until we update X-Rays to use Lib
- (let [query (update query :dataset_query #_{:clj-kondo/ignore [:discouraged-var]} lib/->legacy-MBQL)
- source (source query)
+ (let [source (source query)
root {:database (:database-id query), :source source}]
{:entity query
:source source
:database (:database-id query)
- :query-filter (magic.util/do-with-legacy-query (:dataset_query query) get-in [:query :filter])
+ :query-filter (lib/filters (:dataset_query query))
:full-name (cond
(native-query? query) (tru "Native query")
(table-like? query) (-> source ->root :full-name)
@@ -380,18 +371,19 @@
x)
(m/update-existing :visualization #(instantiate-visualization % bindings available-metrics))))
-(defn- singular-cell-dimension-field-ids
+(mu/defn- singular-cell-dimension-field-ids :- [:maybe [:set [:or :string ::lib.schema.id/field]]]
"Return the set of ids referenced in a cell query"
- [{:keys [cell-query]}]
- (letfn [(collect-dimensions [[op & args]]
- (case (some-> op qp.util/normalize-token)
- :and (mapcat collect-dimensions args)
- := (magic.util/collect-field-references args)
+ [{:keys [cell-query], :as _root} :- ::ads/root]
+ (letfn [(collect-dimensions [[tag _opts & args]]
+ ;; TODO (Cam 10/21/25) -- piccing apart MBQL clauses like this is a little icky and unidiomatic, we really
+ ;; don't discourage digging around in MBQL outside of Lib -- FIXME
+ (case (some-> tag keyword)
+ :and (mapcat collect-dimensions args)
+ (:between :=) (magic.util/collect-field-references args)
nil))]
- (->> cell-query
- collect-dimensions
- (map magic.util/field-reference->id)
- set)))
+ (into #{}
+ (map magic.util/field-reference->id)
+ (collect-dimensions cell-query))))
(defn- matching-dashboard-templates
"Return matching dashboard templates ordered by specificity.
@@ -467,8 +459,7 @@
{:source (assoc source :fields (table->fields source))
:root root
:tables (map #(assoc % :fields (table->fields %)) tables)
- :query-filter (filters/inject-refinement (:query-filter root)
- (:cell-query root))}))
+ :query-filter (filters/inject-refinement (:query-filter root) (:cell-query root))}))
(defn- make-dashboard
([root dashboard-template]
@@ -493,7 +484,7 @@
:as dashboard-template}
{grounded-dimensions :dimensions
grounded-metrics :metrics
- grounded-filters :filters}]
+ grounded-filters :filters} :- ::ads/grounded-values]
(let [card-templates (interesting/normalize-seq-of-maps :card template-cards)
dashcards (combination/grounded-metrics->dashcards
base-context
@@ -695,7 +686,7 @@
"Produce a fully-populated dashboard from the base context for an item and a dashboard template."
[{{:keys [show url query-filter] :as root} :root :as base-context} :- ::ads/context
{:as dashboard-template}
- {grounded-dimensions :dimensions :as grounded-values}]
+ {grounded-dimensions :dimensions :as grounded-values} :- ::ads/grounded-values]
(let [show (or show max-cards)
dashboard (generate-base-dashboard base-context dashboard-template grounded-values)]
(-> dashboard
@@ -731,7 +722,33 @@
:filter-specs template-filters})]
(generate-dashboard base-context template grounded-values)))
-(defmulti automagic-analysis
+(mr/def ::automagic-analysis.opts
+ [:maybe
+ [:map
+ {:closed true}
+ [:cell-query {:optional true} [:maybe ::ads/root.cell-query]]
+ [:show {:optional true} [:maybe [:or pos-int? [:= :all]]]]
+ [:source {:optional true} ::ads/source]
+ [:query-filter {:optional true} [:maybe
+ [:schema
+ {:decode/normalize (fn [filters]
+ (when (seq filters)
+ (if (sequential? (first filters))
+ filters
+ [filters])))}
+ [:sequential ::ads/filter-clause]]]]
+ [:database {:optional true} ::lib.schema.id/database]
+ [:comparison? {:optional true} [:maybe :boolean]]
+ [:rules-prefix {:optional true} [:maybe [:sequential :string]]]
+ ;; TODO -- use [[metabase.xrays.api.automagic-dashboards/DashboardTemplate]] (move it somewhere first)
+ [:dashboard-template {:optional true} [:maybe [:sequential :string]]]]])
+
+(defmulti ^:private automagic-analysis-method
+ {:arglists '([entity opts])}
+ (fn [entity _]
+ (mi/model entity)))
+
+(defn automagic-analysis
"Create a transient dashboard analyzing given entity.
This function eventually calls out to `automagic-dashboard` with two primary arguments:
@@ -739,46 +756,47 @@
passed through the `->root` function, which is an aggregate including the original entity, its
source, what dashboard template categories to apply, etc.
- Additional options such as how many cards to show, a cell query (a drill through), etc."
- {:arglists '([entity opts])}
- (fn [entity _]
- (mi/model entity)))
+ [entity opts]
+ (automagic-analysis-method
+ (lib/normalize ::ads/root.entity entity)
+ (lib/normalize ::automagic-analysis.opts opts)))
-(defmethod automagic-analysis :model/Table
+(defmethod automagic-analysis-method :model/Table
[table opts]
(automagic-dashboard (merge (->root table) opts)))
-(defmethod automagic-analysis :model/Segment
- [segment opts]
+(mu/defmethod automagic-analysis-method :model/Segment
+ [segment :- ::segments.schema/segment
+ opts]
(automagic-dashboard (merge (->root segment) opts)))
-(mu/defmethod automagic-analysis :xrays/Metric
+(mu/defmethod automagic-analysis-method :xrays/Metric
[metric :- ::ads/metric
opts]
(automagic-dashboard (merge (->root metric) opts)))
-(mu/defn- collect-metrics :- [:maybe [:sequential (ms/InstanceOf :xrays/Metric)]]
+(mu/defn- collect-metrics :- [:maybe [:sequential [:and
+ (ms/InstanceOf :xrays/Metric)
+ ::ads/metric]]]
[root :- ::ads/root
question :- [:map
[:dataset_query ::ads/query]]]
- (map (mu/fn [aggregation-clause :- ::mbql.s/Aggregation]
- (if (-> aggregation-clause
- first
- qp.util/normalize-token
- (= :metric))
+ (map (mu/fn [aggregation-clause :- ::lib.schema.aggregation/aggregation]
+ (if (lib/clause-of-type? aggregation-clause :metric)
;; any [:metric ...] MBQL clauses these days are V2 Metrics and Automagic Dashboards do not handle them.
(log/error "X-Rays do not support V2 Metrics.")
(let [table-id (table-id question)]
- (mi/instance :xrays/Metric {:definition {:aggregation [aggregation-clause]
- :source-table table-id}
- :name (names/metric->description root aggregation-clause)
- :table_id table-id}))))
- (magic.util/do-with-legacy-query (:dataset_query question) get-in [:query :aggregation])))
+ (mi/instance :xrays/Metric {:xrays/aggregation aggregation-clause
+ :name (names/metric->description root aggregation-clause)
+ :table-id table-id
+ :xrays/database-id ((some-fn :database-id :database_id) question)}))))
+ (lib/aggregations (:dataset_query question))))
(mu/defn- collect-breakout-fields :- [:maybe [:sequential (ms/InstanceOf :model/Field)]]
[root :- ::ads/root
question :- [:map
[:dataset_query ::ads/query]]]
- (for [breakout (magic.util/do-with-legacy-query (:dataset_query question) get-in [:query :breakout])
+ (for [breakout (lib/breakouts (:dataset_query question))
field-clause (take 1 (magic.util/collect-field-references breakout))
:let [field (magic.util/->field root field-clause)]
:when (and field
@@ -807,23 +825,41 @@
[(collect-metrics root question)
(collect-breakout-fields root question)])))
-(defn- preserve-entity-element
+(mu/defn- preserve-entity-element
"Ensure that elements of an original dataset query are preserved in dashcard queries."
[dashboard
entity
- entity-element]
- (if-let [element-value (magic.util/do-with-legacy-query (:dataset_query entity) get-in [:query entity-element])]
- (letfn [(splice-element [dashcard]
- (m/update-existing-in dashcard [:card :dataset_query]
- magic.util/do-with-legacy-query
- update-in [:query entity-element]
- (fnil into (empty element-value))
- element-value))]
- (update dashboard :dashcards (partial map splice-element)))
- dashboard))
-
-(defn- query-based-analysis
- [{:keys [entity] :as root} opts {:keys [cell-query cell-url]}]
+ getter-fn
+ setter-fn]
+ (binding [lib.schema/*HACK-disable-ref-validation* true]
+ (if-let [element-value (some-> entity :dataset_query not-empty getter-fn)]
+ (letfn [(splice-element [dashcard]
+ (m/update-existing-in dashcard [:card :dataset_query] setter-fn element-value))]
+ (update dashboard :dashcards (partial mapv splice-element)))
+ dashboard)))
+
+(defn- preserve-joins [dashboard entity]
+ (preserve-entity-element dashboard entity lib/joins (fn [query joins]
+ (reduce lib/join query joins))))
+
+(defn- preserve-expressions [dashboard entity]
+ (preserve-entity-element dashboard entity lib/expressions (fn [query expressions]
+ (reduce
+ (fn [query expression]
+ (lib/expression query
+ (:lib/expression-name (lib/options expression))
+ expression))
+ query
+ expressions))))
+
+(mu/defn- query-based-analysis
+ [{:keys [entity] :as root} :- ::ads/root
+ opts :- ::automagic-analysis.opts
+ {:keys [cell-query cell-url]} :- [:maybe
+ [:map
+ {:closed true}
+ [:cell-query {:optional true} [:maybe ::ads/root.cell-query]]
+ [:cell-url :string]]]]
(let [transient-dash (if (table-like? entity)
(let [root' (merge root
(when cell-query
@@ -847,11 +883,12 @@
{:transient_name title
:name title})))))]
(-> transient-dash
- (preserve-entity-element (:entity root) :joins)
- (preserve-entity-element (:entity root) :expressions))))
+ (preserve-joins (:entity root))
+ (preserve-expressions (:entity root)))))
-(defmethod automagic-analysis :model/Card
- [card {:keys [cell-query] :as opts}]
+(mu/defmethod automagic-analysis-method :model/Card
+ [card
+ {:keys [cell-query] :as opts} :- ::automagic-analysis.opts]
(let [root (->root card)
cell-url (format "%squestion/%s/cell/%s" public-endpoint
(u/the-id card)
@@ -861,25 +898,22 @@
{:cell-query cell-query
:cell-url cell-url}))))
-(defmethod automagic-analysis :model/Query
- [query {:keys [cell-query] :as opts}]
- (let [root (->root query)
- cell-query (when cell-query
- #_{:clj-kondo/ignore [:deprecated-var]}
- (mbql.normalize/normalize-fragment [:query :filter] cell-query))
- opts (cond-> opts
- cell-query (assoc :cell-query cell-query))
- cell-url (format "%sadhoc/%s/cell/%s" public-endpoint
- (magic.util/encode-base64-json (:dataset_query query))
- (magic.util/encode-base64-json cell-query))]
- (query-based-analysis root opts
- (when cell-query
- {:cell-query cell-query
- :cell-url cell-url}))))
+(mu/defmethod automagic-analysis-method :model/Query
+ [query
+ {:keys [cell-query] :as opts} :- ::automagic-analysis.opts]
+ (let [root (->root query)
+ cell-url (format "%sadhoc/%s/cell/%s" public-endpoint
+ (magic.util/encode-base64-json (:dataset_query query))
+ (magic.util/encode-base64-json cell-query))]
+ (binding [lib.schema/*HACK-disable-ref-validation* true]
+ (query-based-analysis root opts
+ (when cell-query
+ {:cell-query cell-query
+ :cell-url cell-url})))))
-(mu/defmethod automagic-analysis :model/Field
+(mu/defmethod automagic-analysis-method :model/Field
[field :- ::ads/field
- opts]
+ opts :- ::automagic-analysis.opts]
(automagic-dashboard (merge (->root field) opts)))
(defn- load-tables-with-enhanced-table-stats
diff --git a/src/metabase/xrays/automagic_dashboards/filters.clj b/src/metabase/xrays/automagic_dashboards/filters.clj
index e498511e9ed..a9e12855320 100644
--- a/src/metabase/xrays/automagic_dashboards/filters.clj
+++ b/src/metabase/xrays/automagic_dashboards/filters.clj
@@ -1,8 +1,8 @@
(ns metabase.xrays.automagic-dashboards.filters
(:require
- ;; legacy usage, do not use this in new code
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]
- ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.legacy-mbql.util :as mbql.u]
+ [metabase.lib.core :as lib]
+ [metabase.lib.equality :as lib.equality]
+ [metabase.lib.schema :as lib.schema]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.malli :as mu]
@@ -121,7 +121,8 @@
"Add up to `max-filters` filters to dashboard `dashboard`. The `dimensions` argument is a list of fields for which to
create filters."
[dashboard :- ::ads/dashboard
- dimensions max-filters]
+ dimensions
+ max-filters]
(let [fks (when-let [table-ids (not-empty (set (keep (comp :table_id :card)
(:dashcards dashboard))))]
(field/with-targets (t2/select :model/Field
@@ -149,25 +150,7 @@
dashboard)))
dashboard))))
-(defn- flatten-filter-clause
- "Returns a sequence of filter subclauses making up `filter-clause` by flattening `:and` compound filters.
-
- (flatten-filter-clause [:and
- [:= [:field 1 nil] 2]
- [:and
- [:= [:field 3 nil] 4]
- [:= [:field 5 nil] 6]]])
- ;; -> ([:= [:field 1 nil] 2]
- [:= [:field 3 nil] 4]
- [:= [:field 5 nil] 6])"
- [[clause-name, :as filter-clause]]
- (when (seq filter-clause)
- (if (= clause-name :and)
- #_{:clj-kondo/ignore [:deprecated-var]}
- (rest (mbql.u/simplify-compound-filter filter-clause))
- [filter-clause])))
-
-(defn inject-refinement
+(mu/defn inject-refinement :- [:maybe ::lib.schema/filters]
"Inject a filter refinement into an MBQL filter clause, returning a new filter clause.
There are two reasons why we want to do this: 1) to reduce visual noise when we display applied filters; and 2) some
@@ -177,17 +160,19 @@
of the latter. Therefore we can rewrite the combined clause to ommit the more broad version from the main clause.
Assumes both filter clauses can be flattened by recursively merging `:and` claueses
(ie. no `:and`s inside `:or` or `:not`)."
- [filter-clause refinement]
- (let [in-refinement? (into #{}
- (map magic.util/collect-field-references)
- (flatten-filter-clause refinement))
- existing-filters (->> filter-clause
- flatten-filter-clause
- (remove (comp in-refinement? magic.util/collect-field-references)))]
- (if (seq existing-filters)
- ;; since the filters are programatically generated they won't have passed thru normalization, so make sure we
- ;; normalize them before passing them to `combine-filter-clauses`, which validates its input
- #_{:clj-kondo/ignore [:deprecated-var]}
- (apply mbql.u/combine-filter-clauses (map (partial mbql.normalize/normalize-fragment [:query :filter])
- (cons refinement existing-filters)))
- refinement)))
+ [filter-clauses :- [:maybe ::lib.schema/filters]
+ refinement :- [:maybe vector?]]
+ (if (some? refinement)
+ ;; normalize refinement since it's read from the YAML files or whatever
+ (let [refinement (lib/normalize refinement)
+ ;; TODO (Cam 10/21/2025) -- HACK -- I wanted to use [[lib/filter-parts]] for this but we need to pass in
+ ;; query as part of this. Maybe we can refactor this code so we can use that, or just add a new function to
+ ;; see if we have existing filters against a column.
+ refinement-column (nth refinement 2)
+ ;; remove any existing filters against the column in the refinement filter.
+ filter-clauses' (into []
+ (remove (fn [a-filter]
+ (lib.equality/= (nth a-filter 2) refinement-column)))
+ (lib/simplify-filters filter-clauses))]
+ (lib/simplify-filters (conj filter-clauses' refinement)))
+ filter-clauses))
diff --git a/src/metabase/xrays/automagic_dashboards/interesting.clj b/src/metabase/xrays/automagic_dashboards/interesting.clj
index f6e0d771767..26a971e4759 100644
--- a/src/metabase/xrays/automagic_dashboards/interesting.clj
+++ b/src/metabase/xrays/automagic_dashboards/interesting.clj
@@ -41,13 +41,11 @@
(:require
[clojure.math.combinatorics :as math.combo]
[clojure.string :as str]
- [clojure.walk :as walk]
[java-time.api :as t]
[medley.core :as m]
- ;; legacy usages, do not use legacy MBQL stuff in new code.
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]
- ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.legacy-mbql.util :as mbql.u]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.schema.ref :as lib.schema.ref]
[metabase.models.interface :as mi]
[metabase.util :as u]
@@ -91,27 +89,28 @@
(can-use? :hour) :hour))
(if (can-use? :day) :day :hour))))
-(mu/defmethod ->reference [:mbql :model/Field] :- ::mbql.s/field
- [_ {:keys [fk_target_field_id id link aggregation name base_type] :as field}]
- (let [reference (mbql.normalize/normalize
- (cond
- link [:field id {:source-field link}]
- fk_target_field_id [:field fk_target_field_id {:source-field id}]
- id [:field id {:base-type base_type}]
- :else [:field name {:base-type base_type}]))]
- ;; legacy usages -- use Lib in new code going forward
- #_{:clj-kondo/ignore [:deprecated-var]}
- (cond
- (isa? base_type :type/Temporal)
- (mbql.u/with-temporal-unit reference (keyword (or aggregation
- (optimal-temporal-resolution field))))
+(mu/defn field->metadata :- ::lib.schema.metadata/column
+ "Convert a Field from the app DB to Lib column metadata, and do a bunch of weird additional transformations that
+ I (Cam) do not really understand (see below).
+
+ TODO (Cam 10/21/25) -- we wouldn't need to do this at all if we further reworked the X-Ray code to either fetch
+ `:metadata/column`s directly or use Metadata Providers."
+ [{fk-target-field-id :fk_target_field_id, base-type :base_type, :keys [id link aggregation], :as field} :- (ms/InstanceOf :model/Field)]
+ (let [col (if fk-target-field-id
+ (-> (t2/select-one :metadata/column :id fk-target-field-id)
+ (assoc :fk-field-id id, :lib/source :source/implicitly-joinable))
+ (lib-be/instance->metadata field :metadata/column))]
+ (cond-> col
+ link
+ (assoc :fk-field-id link, :lib/source :source/implicitly-joinable)
+
+ (isa? base-type :type/Temporal)
+ (lib/with-temporal-bucket (keyword (or aggregation
+ (optimal-temporal-resolution field))))
(and aggregation
- (isa? base_type :type/Number))
- (mbql.u/update-field-options reference assoc-in [:binning :strategy] (keyword aggregation))
-
- :else
- reference)))
+ (isa? base-type :type/Number))
+ (lib/with-binning {:strategy (keyword aggregation)}))))
(defmethod ->reference [:string :model/Field]
[_ {:keys [display_name full-name link]}]
@@ -131,10 +130,10 @@
(or full-name name))
(defmethod ->reference [:mbql :xrays/Metric]
- [_ {:keys [id definition]}]
+ [_ {:keys [id], :as metric}]
(if id
[:metric id]
- (-> definition :aggregation first)))
+ (:xrays/aggregation metric)))
(defmethod ->reference [:native :model/Field]
[_ field]
@@ -150,44 +149,43 @@
(map? form) ((some-fn :full-name :name) form))
form))
-(defn transform-metric-aggregate
+(mu/defn transform-metric-aggregate :- ::ads/external-op
"Map a metric aggregate definition from nominal types to semantic types."
- [m decoder]
- (walk/prewalk
- (fn [v]
- (if (vector? v)
- (let [[d n] v]
- (if (= "dimension" d)
- (decoder n)
- v))
- v))
- m))
-
-(mu/defn ground-metric :- [:sequential ads/grounded-metric]
+ [[ag-type & args] dimension-name->field]
+ {:lib/type :lib/external-op
+ :operator (keyword ag-type)
+ :args (mapv (fn [arg]
+ (if (and (vector? arg)
+ (= (first arg) "dimension"))
+ (when-let [field (dimension-name->field (second arg))]
+ (field->metadata field))
+ arg))
+ args)})
+
+(mu/defn ground-metric :- [:sequential ::ads/grounded-metric]
"Generate \"grounded\" metrics from the mapped dimensions (dimension name -> field matches).
Since there may be multiple matches to a dimension, this will produce a sequence of potential matches."
[{metric-name :metric-name
metric-score :score
- metric-definition :metric} :- ads/normalized-metric-template
- ground-dimensions :- ads/dim-name->matching-fields]
+ metric-definition :metric} :- ::ads/normalized-metric-template
+ ground-dimensions :- ::ads/dim-name->matching-fields]
(let [named-dimensions (dashboard-templates/collect-dimensions metric-definition)]
(->> (map (comp :matches ground-dimensions) named-dimensions)
(apply math.combo/cartesian-product)
(map (partial zipmap named-dimensions))
(map (fn [nm->field]
- (let [xform (update-vals nm->field (partial ->reference :mbql))]
- {:metric-name metric-name
- :metric-title metric-name
- :metric-score metric-score
- :metric-definition {:aggregation
- [(transform-metric-aggregate metric-definition xform)]}
- ;; Required for title interpolation in grounded-metrics->dashcards
- :dimension-name->field nm->field}))))))
-
-(mu/defn grounded-metrics :- [:sequential ads/grounded-metric]
+ {:metric-name metric-name
+ :metric-title metric-name
+ :metric-score metric-score
+ :metric-definition {:xrays/aggregations
+ [(transform-metric-aggregate metric-definition nm->field)]}
+ ;; Required for title interpolation in grounded-metrics->dashcards
+ :dimension-name->field nm->field})))))
+
+(mu/defn grounded-metrics :- [:sequential ::ads/grounded-metric]
"Given a set of metric definitions and grounded (assigned) dimensions, produce a sequence of grounded metrics."
- [metric-templates :- [:sequential ads/normalized-metric-template]
- ground-dimensions :- ads/dim-name->matching-fields]
+ [metric-templates :- [:sequential ::ads/normalized-metric-template]
+ ground-dimensions :- ::ads/dim-name->matching-fields]
(mapcat #(ground-metric % ground-dimensions) metric-templates))
(defn normalize-seq-of-maps
@@ -342,7 +340,7 @@
(let [scored-bindings (score-bindings candidate-binding-values)]
(second (last (sort-by first scored-bindings)))))
-(mu/defn find-dimensions :- ads/dim-name->dim-defs+matches
+(mu/defn find-dimensions :- ::ads/dim-name->dim-defs+matches
"Bind fields to dimensions from the dashboard template and resolve overloaded cases in which multiple fields match the
dimension specification.
@@ -351,7 +349,7 @@
(see `most-specific-definition` for details).
The context is passed in, but it only needs tables and fields in `candidate-bindings`. It is not extensively used."
- [context dimension-specs :- [:maybe [:sequential ads/dimension-template]]]
+ [context dimension-specs :- [:maybe [:sequential ::ads/dimension-template]]]
(->> (candidate-bindings context dimension-specs)
(map (comp most-specific-matched-dimension val))
(apply merge-with (fn [a b]
@@ -392,7 +390,7 @@
(add-field-links-to-definitions (enriched-field-with-sources context entity))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(defn grounded-filters
+(mu/defn grounded-filters :- [:sequential ::ads/grounded-filter]
"Take filter templates (as from a dashboard template's :filters) and ground dimensions and produce a map of the
filter name to grounded versions of the filter."
[filter-templates ground-dimensions]
@@ -401,40 +399,36 @@
(let [[fname {:keys [filter] :as v}] (first fltr)
dims (dashboard-templates/collect-dimensions v)
opts (->> (map (comp
- (partial map (partial ->reference :mbql))
:matches
ground-dimensions) dims)
(apply math.combo/cartesian-product)
(map (partial zipmap dims)))]
- (seq (for [opt opts
- :let [f
- (walk/prewalk
- (fn [x]
- (if (vector? x)
- (let [[ds dim-name] x]
- (if (and (= "dimension" ds)
- (string? dim-name))
- (opt dim-name)
- x))
- x))
- filter)]]
+ (seq (for [opt opts
+ :let [[op & args] filter
+ f
+ {:lib/type :lib/external-op
+ :operator (keyword op)
+ :args (mapv (fn [arg]
+ (if (and (vector? arg)
+ (= (first arg) "dimension"))
+ (when-let [field (opt (second arg))]
+ (field->metadata field))
+ arg))
+ args)}]]
(assoc v :filter f :filter-name fname))))))
flatten))
-(mu/defn identify
- :- [:map
- [:dimensions ads/dim-name->matching-fields]
- [:metrics [:sequential ads/grounded-metric]]]
+(mu/defn identify :- ::ads/grounded-values
"Identify interesting metrics and dimensions of a `thing`. First identifies interesting dimensions, and then
interesting metrics which are satisfied.
Metrics from the template are assigned a score of 50; user defined metrics a score of 95"
- [context
+ [context :- ::ads/context
{:keys [dimension-specs
metric-specs
filter-specs]} :- [:map
- [:dimension-specs [:maybe [:sequential ads/dimension-template]]]
- [:metric-specs [:maybe [:sequential ads/metric-template]]]
- [:filter-specs [:maybe [:sequential ads/filter-template]]]]]
+ [:dimension-specs [:maybe [:sequential ::ads/dimension-template]]]
+ [:metric-specs [:maybe [:sequential ::ads/metric-template]]]
+ [:filter-specs [:maybe [:sequential ::ads/filter-template]]]]]
(let [dims (->> (find-dimensions context dimension-specs)
(add-field-self-reference context))
metrics (-> (normalize-seq-of-maps :metric metric-specs)
@@ -448,6 +442,6 @@
(when (mi/instance-of? :xrays/Metric entity)
[{:metric-name "this"
:metric-title (:name entity)
- :metric-definition {:aggregation [(->reference :mbql entity)]}
+ :metric-definition {:xrays/aggregations [(:xrays/aggregation entity)]}
:metric-score dashboard-templates/max-score}])))
:filters (grounded-filters filter-specs dims)}))
diff --git a/src/metabase/xrays/automagic_dashboards/names.clj b/src/metabase/xrays/automagic_dashboards/names.clj
index 7ddce36d57f..db2ac742451 100644
--- a/src/metabase/xrays/automagic_dashboards/names.clj
+++ b/src/metabase/xrays/automagic_dashboards/names.clj
@@ -2,12 +2,12 @@
(:require
[clojure.string :as str]
[java-time.api :as t]
- ;; legacy usages, do not use legacy MBQL stuff in new code.
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]
- [metabase.lib.util.match :as lib.util.match]
- [metabase.query-processor.util :as qp.util]
+ [metabase.lib.core :as lib]
+ [metabase.lib.schema.aggregation :as lib.schema.aggregation]
+ [metabase.lib.schema.expression :as lib.schema.expression]
+ [metabase.lib.schema.ref :as lib.schema.ref]
[metabase.util.date-2 :as u.date]
- [metabase.util.i18n :refer [deferred-tru tru]]
+ [metabase.util.i18n :as i18n :refer [deferred-tru tru]]
[metabase.util.malli :as mu]
[metabase.util.time :as u.time]
[metabase.xrays.automagic-dashboards.schema :as ads]
@@ -26,36 +26,46 @@
:cum-count (deferred-tru "cumulative count")
:cum-sum (deferred-tru "cumulative sum")})
-(defn metric-name
+(mu/defn metric-name :- ::ads/string-or-18n-string
"Return the name of the metric or name by describing it."
- [[op & args :as metric]]
- (if (magic.util/adhoc-metric? metric)
- (-> op qp.util/normalize-token op->name)
- (second args)))
+ [[tag :as metric] :- vector?]
+ (cond
+ (magic.util/adhoc-metric? metric)
+ (-> tag keyword op->name)
-(defn- join-enumeration
+ (lib/clause-of-type? metric :field)
+ (lib/field-ref-name metric)
+
+ :else
+ (throw (ex-info (format "Don't know how to get the name of %s" (pr-str metric))
+ {:metric metric}))))
+
+(mu/defn- join-enumeration :- ::ads/string-or-18n-string
"Join a sequence as [1 2 3 4] to \"1, 2, 3 and 4\""
- [xs]
+ [xs :- [:sequential :any]]
(if (next xs)
(tru "{0} and {1}" (str/join ", " (butlast xs)) (last xs))
- (first xs)))
+ (str (first xs))))
(def ^{:arglists '([root])} source-name
"Return the (display) name of the source of a given root object."
(comp (some-fn :display_name :name) :source))
-(defn metric->description
+(mu/defn metric->description :- [:or :string [:fn {:error/message "localized string"} i18n/localized-string?]]
"Return a description for the metric."
- [root aggregation-clause]
+ [root :- ::ads/root
+ aggregation-clause :- [:or
+ ::lib.schema.aggregation/aggregation
+ ::lib.schema.aggregation/aggregations]]
(join-enumeration
(for [metric (if (sequential? (first aggregation-clause))
aggregation-clause
[aggregation-clause])]
(if (magic.util/adhoc-metric? metric)
- (tru "{0} of {1}" (metric-name metric) (or (some->> metric
- second
- (magic.util/->field root)
- :display_name)
+ (tru "{0} of {1}" (metric-name metric) (or (when (> (count metric) 2)
+ (->> (nth metric 2) ; icky
+ (magic.util/->field root)
+ :display_name))
(source-name root)))
(metric-name metric)))))
@@ -64,12 +74,15 @@
[root :- ::ads/root
question :- [:map
[:dataset_query ::ads/query]]]
- (let [aggregations (->> (magic.util/do-with-legacy-query (:dataset_query question) get-in [:query :aggregation])
+ (let [aggregations (->> (lib/aggregations (:dataset_query question))
(metric->description root))
- dimensions (->> (magic.util/do-with-legacy-query (:dataset_query question) get-in [:query :breakout])
- (mapcat magic.util/collect-field-references)
- (map (partial magic.util/->field root))
- (map :display_name)
+ field-ids (into
+ #{}
+ (mapcat lib/all-field-ids)
+ (lib/breakouts (:dataset_query question)))
+ dimensions (->> field-ids
+ (mapv (partial magic.util/->field root))
+ (mapv :display_name)
join-enumeration)]
(if dimensions
(tru "{0} by {1}" aggregations dimensions)
@@ -77,8 +90,8 @@
(defmulti ^:private humanize-filter-value
{:arglists '([root mbql-clause])}
- (fn [_root [tag]]
- (qp.util/normalize-token tag)))
+ (fn [_root mbql-clause]
+ (lib/dispatch-value mbql-clause)))
(def ^:private unit-name (comp {:minute-of-hour (deferred-tru "minute")
:hour-of-day (deferred-tru "hour")
@@ -89,28 +102,23 @@
:month-of-year (deferred-tru "month")
:quarter-of-year (deferred-tru "quarter")
:year (deferred-tru "year")}
- qp.util/normalize-token))
+ keyword))
-(mu/defn- item-reference->field
+(mu/defn- item-reference->field :- [:map
+ [:item-name :string]]
"Turn a field reference into a field."
- [root :- ::ads/root
- [item-type :as item-reference]]
- (case item-type
- (:field "field") (let [normalized-field-reference (mbql.normalize/normalize item-reference)
- temporal-unit (lib.util.match/match-one normalized-field-reference
- [:field _ (opts :guard :temporal-unit)]
- (:temporal-unit opts))
- {:keys [display_name] :as field-record} (cond-> (->> normalized-field-reference
- magic.util/collect-field-references
- first
- (magic.util/->field root))
- temporal-unit
- (assoc :unit temporal-unit))
- item-name (cond->> display_name
- (some-> temporal-unit u.date/extract-units)
- (tru "{0} of {1}" (unit-name temporal-unit)))]
- (assoc field-record :item-name item-name))
- (:expression "expression") {:item-name (second item-reference)}
+ [root :- ::ads/root
+ [tag _opts x :as a-ref] :- ::lib.schema.ref/ref]
+ (case tag
+ :field (let [temporal-unit (lib/raw-temporal-bucket a-ref)
+ {:keys [display_name] :as field-record} (cond-> (magic.util/->field root a-ref)
+ temporal-unit
+ (assoc :unit temporal-unit))
+ item-name (cond->> display_name
+ (some-> temporal-unit u.date/extract-units)
+ (tru "{0} of {1}" (unit-name temporal-unit)))]
+ (assoc field-record :item-name item-name))
+ :expression {:item-name x}
{:item-name "item"}))
(defn item-name
@@ -163,48 +171,48 @@
:week-of-year) (u.date/extract dt unit))))
(mu/defmethod humanize-filter-value :=
- [root :- ::ads/root
- [_ field-reference value]]
+ [root :- ::ads/root
+ [_ _opts field-reference value] :- :mbql.clause/=]
(let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)]
(if (isa? (or effective_type base_type) :type/Temporal)
(tru "{0} is {1}" item-name (humanize-datetime value unit))
(tru "{0} is {1}" item-name value))))
(mu/defmethod humanize-filter-value :>=
- [root :- ::ads/root
- [_ field-reference value]]
+ [root :- ::ads/root
+ [_ _opts field-reference value] :- :mbql.clause/>=]
(let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)]
(if (isa? (or effective_type base_type) :type/Temporal)
(tru "{0} is not before {1}" item-name (humanize-datetime value unit))
(tru "{0} is at least {1}" item-name value))))
(mu/defmethod humanize-filter-value :>
- [root :- ::ads/root
- [_ field-reference value]]
+ [root :- ::ads/root
+ [_ _opts field-reference value] :- :mbql.clause/>]
(let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)]
(if (isa? (or effective_type base_type) :type/Temporal)
(tru "{0} is after {1}" item-name (humanize-datetime value unit))
(tru "{0} is greater than {1}" item-name value))))
(mu/defmethod humanize-filter-value :<=
- [root :- ::ads/root
- [_ field-reference value]]
+ [root :- ::ads/root
+ [_ _opts field-reference value] :- :mbql.clause/<=]
(let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)]
(if (isa? (or effective_type base_type) :type/Temporal)
(tru "{0} is not after {1}" item-name (humanize-datetime value unit))
(tru "{0} is no more than {1}" item-name value))))
(mu/defmethod humanize-filter-value :<
- [root :- ::ads/root
- [_ field-reference value]]
+ [root :- ::ads/root
+ [_ _opts field-reference value] :- :mbql.clause/<]
(let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)]
(if (isa? (or effective_type base_type) :type/Temporal)
(tru "{0} is before {1}" item-name (humanize-datetime value unit))
(tru "{0} is less than {1}" item-name value))))
(mu/defmethod humanize-filter-value :between
- [root :- ::ads/root
- [_ field-reference min-value max-value]]
+ [root :- ::ads/root
+ [_ _opts field-reference min-value max-value] :- :mbql.clause/between]
(let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)]
(or (when (isa? (or effective_type base_type) :type/Temporal)
(let [humanized-min (humanize-datetime min-value unit)
@@ -214,32 +222,33 @@
(tru "{0} is between {1} and {2}" item-name min-value max-value))))
(mu/defmethod humanize-filter-value :inside
- [root :- ::ads/root
- [_ lat-reference lon-reference lat-max lon-min lat-min lon-max]]
+ [root :- ::ads/root
+ [_ _opts lat-reference lon-reference lat-max lon-min lat-min lon-max] :- :mbql.clause/inside]
(tru "{0} is between {1} and {2}; and {3} is between {4} and {5}"
(item-name root lon-reference) lon-min lon-max
(item-name root lat-reference) lat-min lat-max))
(mu/defmethod humanize-filter-value :and
- [root :- ::ads/root
- [_ & clauses]]
+ [root :- ::ads/root
+ [_ _opts & clauses] :- :mbql.clause/and]
(->> clauses
(map (partial humanize-filter-value root))
join-enumeration))
(mu/defmethod humanize-filter-value :default
- [root :- ::ads/root
- [_ field-reference value]]
+ [root :- ::ads/root
+ [_ field-reference value] :- ::lib.schema.expression/boolean]
(let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)]
(if (isa? (or effective_type base_type) :type/Temporal)
(tru "{0} relates to {1}" item-name (humanize-datetime value unit))
(tru "{0} relates to {1}" item-name value))))
-(defn cell-title
+(mu/defn cell-title :- :string
"Return a cell title given a root object and a cell query."
- [root cell-query]
+ [root :- ::ads/root
+ cell-query :- ::ads/root.cell-query]
(str/join " " [(if-let [aggregation (-> (get-in root [:entity :dataset_query])
- (magic.util/do-with-legacy-query get-in [:query :aggregation]))]
+ lib/aggregations)]
(metric->description root aggregation)
(:full-name root))
(tru "where {0}" (humanize-filter-value root cell-query))]))
diff --git a/src/metabase/xrays/automagic_dashboards/populate.clj b/src/metabase/xrays/automagic_dashboards/populate.clj
index ffe69de6767..4fb6e62932b 100644
--- a/src/metabase/xrays/automagic_dashboards/populate.clj
+++ b/src/metabase/xrays/automagic_dashboards/populate.clj
@@ -140,7 +140,7 @@
{query :dataset_query, :keys [title description width height id] :as dashcard} :- ::ads/card-template
[x y]]
(let [query-fields (when query
- (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]
+ (binding [lib.schema/*HACK-disable-ref-validation* true]
(-> {:dataset_query (lib-be/normalize-query query)}
queries/populate-card-query-fields
(select-keys [:query_type :database_id :table_id]))))
@@ -320,7 +320,7 @@
(mu/defn- create-dashboard-populate-dashcards :- [:sequential ::ads/dashcard]
[dashcards :- [:maybe [:sequential ::ads/card-template]]]
- (let [card-id->can-run-adhoc-query (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]
+ (let [card-id->can-run-adhoc-query (binding [lib.schema/*HACK-disable-ref-validation* true]
(into {}
(map (juxt ::id :can_run_adhoc_query))
(queries/with-can-run-adhoc-query
diff --git a/src/metabase/xrays/automagic_dashboards/schema.clj b/src/metabase/xrays/automagic_dashboards/schema.clj
index b7a58d6b0ee..36bdd185f76 100644
--- a/src/metabase/xrays/automagic_dashboards/schema.clj
+++ b/src/metabase/xrays/automagic_dashboards/schema.clj
@@ -1,19 +1,48 @@
(ns metabase.xrays.automagic-dashboards.schema
(:require
[malli.core :as mc]
- [malli.util :as mut]
- ;; legacy usages, do not use legacy MBQL stuff in new code.
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]
[metabase.lib.core :as lib]
[metabase.lib.schema :as lib.schema]
+ [metabase.lib.schema.aggregation :as lib.schema.aggregation]
[metabase.lib.schema.common :as lib.schema.common]
+ [metabase.lib.schema.expression :as lib.schema.expression]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
+ [metabase.util.i18n :as i18n]
[metabase.util.malli.registry :as mr]
- [metabase.util.malli.schema :as ms]))
+ [metabase.util.malli.schema :as ms]
+ [toucan2.core :as t2]))
+
+(mr/def ::string-or-18n-string
+ [:or :string [:fn {:error/message "localized string"} i18n/localized-string?]])
+
+(mr/def ::root.entity
+ [:multi
+ {:dispatch t2/model}
+ [:xrays/Metric [:ref ::metric]]
+ [::mc/default [:map
+ [:name {:optional true} :string]]]])
+
+(mr/def ::filter-clause
+ [:and
+ {:decode/normalize (fn [x]
+ (when (sequential? x)
+ (if (map? (second x))
+ x
+ (lib/->pMBQL x))))}
+ [:ref ::lib.schema.mbql-clause/clause]
+ [:ref ::lib.schema.expression/boolean]])
+
+(mr/def ::root.cell-query
+ ::filter-clause)
(mr/def ::root
[:map
- [:database ::lib.schema.id/database]])
+ [:database ::lib.schema.id/database]
+ [:entity {:optional true} [:ref ::root.entity]]
+ [:query-filter {:optional true} [:maybe [:sequential ::filter-clause]]]
+ [:cell-query {:optional true} [:maybe [:ref ::root.cell-query]]]])
(mr/def ::source
[:or
@@ -27,290 +56,180 @@
(mr/def ::context
"The big ball of mud data object from which we generate x-rays"
- (mc/schema
- [:map
- [:source {:optional true} ::source]
- [:root {:optional true} [:ref ::root]]
- [:tables {:optional true} any?]
- [:query-filter {:optional true} any?]]))
+ [:map
+ [:source {:optional true} ::source]
+ [:root {:optional true} [:ref ::root]]
+ [:tables {:optional true} any?]
+ [:query-filter {:optional true} [:maybe [:sequential ::filter-clause]]]])
(mr/def ::query
- "Schema for the type of MBQL queries handled by X-Rays. Currently, either legacy or MBQL 5 is supported, and functions
- use [[metabase.xrays.automagic-dashboards.util/do-with-legacy-query]]
- or [[metabase.xrays.automagic-dashboards.util/do-with-mbql5-query]] as needed; in the future we can change this to
- only accept MBQL 5 and update any code working with legacy MBQL."
- [:and
+ "Schema for the type of MBQL queries handled by X-Rays."
+ [:ref ::lib.schema/query])
+
+(mr/def ::external-op
+ [:merge
+ ::lib.schema.common/external-op
[:map
- [:database ::lib.schema.id/database]]
- [:multi {:dispatch lib/normalized-mbql-version}
- [:mbql-version/mbql5 [:ref ::lib.schema/query]]
- [:mbql-version/legacy [:ref ::mbql.s/Query]]]])
+ [:args [:sequential [:multi
+ {:dispatch coll?, :error/message "Should be a literal or column metadata"}
+ [false :any]
+ [true ::lib.schema.metadata/column]]]]]])
+
+(mr/def ::aggregation
+ [:or ::lib.schema.aggregation/aggregation ::external-op])
-(mr/def ::table-id-or-database-id
+(mr/def ::metric
+ "Schema for an `:xrays/Metric`."
[:and
[:map
- [:table_id {:optional true} [:maybe ::lib.schema.id/table]]
+ {:closed true}
+ [:name :string]
+ [:xrays/aggregation ::aggregation]
+ [:table-id {:optional true} [:maybe ::lib.schema.id/table]]
[:xrays/database-id {:optional true} [:maybe ::lib.schema.id/database]]]
[:fn
- {:error/message "If instance does not have :table_id, it must have :xrays/database-id"}
- (some-fn :table_id :xrays/database-id)]])
-
-(mr/def ::field
- [:ref ::table-id-or-database-id])
+ {:error/message "If instance does not have :table-id, it must have :xrays/database-id"}
+ (some-fn :table-id :xrays/database-id)]])
-(mr/def ::metric
- [:ref ::table-id-or-database-id])
-
-#_(def dashcard
- "The base unit thing we are trying to produce in x-rays"
- ;; TODO - Beef these specs up, esp. the any?s
- (mc/schema
- [:map
- [:dataset_query {:optional true} [:ref ::query]]
- [:dimensions {:optional true} [:sequential string?]]
- [:group {:optional true} string?]
- [:height pos-int?]
- [:metrics {:optional true} any?]
- [:position {:optional true} nat-int?]
- [:card-score {:optional true} number?]
- [:total-score {:optional true} nat-int?]
- [:metric-score {:optional true} nat-int?]
- [:score-components {:optional true} [:sequential nat-int?]]
- [:title {:optional true} string?]
- [:visualization {:optional true} any?]
- [:width pos-int?]
- [:x_label {:optional true} string?]]))
-
-#_(def dashcards
- "A bunch of dashcards"
- (mc/schema [:maybe [:sequential dashcard]]))
-
-(def field-type
+(mr/def ::field-type
"A dimension reference, as either a semantic type or entity type and semantic type."
- (mc/schema
- [:or
- [:tuple :keyword]
- [:tuple :keyword :keyword]]))
+ [:or
+ [:tuple :keyword]
+ [:tuple :keyword :keyword]])
-;;
-(def dimension-value
+(mr/def ::dimension-value
"A specification for the basic keys in the value of a dimension template."
- (mc/schema
- [:map
- [:field_type field-type]
- [:score {:optional true} nat-int?]
- [:max_cardinality {:optional true} nat-int?]
- [:named {:optional true} [:string {:min 1}]]]))
+ [:map
+ [:field_type ::field-type]
+ [:score {:optional true} nat-int?]
+ [:max_cardinality {:optional true} nat-int?]
+ [:named {:optional true} [:string {:min 1}]]])
-(def dimension-template
+(mr/def ::dimension-template
"A specification for the basic keys in a dimension template."
- (mc/schema
- [:map-of
- {:min 1 :max 1}
- [:string {:min 1}]
- dimension-value]))
+ [:map-of
+ {:min 1 :max 1}
+ [:string {:min 1}]
+ ::dimension-value])
-(def metric-value
+(mr/def ::metric-value
"A specification for the basic keys in the value of a metric template."
- (mc/schema
- [:map
- [:metric [:vector some?]]
- [:score {:optional true} nat-int?]
- ;[:name some?]
- ]))
+ [:map
+ {:closed true}
+ [:metric [:vector some?]]
+ [:score {:optional true} nat-int?]
+ [:name {:optional true} ::string-or-18n-string]])
-(def metric-template
+(mr/def ::metric-template
"A specification for the basic keys in a metric template."
- (mc/schema
- [:map-of
- {:min 1 :max 1}
- [:string {:min 1}]
- metric-value]))
+ [:map-of
+ {:min 1 :max 1}
+ [:string {:min 1}]
+ ::metric-value])
-(def filter-value
+(mr/def ::filter-value
"A specification for the basic keys in the value of a filter template."
- (mc/schema
- [:map
- [:filter [:vector some?]]
- [:score nat-int?]]))
+ [:map
+ [:filter [:vector some?]]
+ [:score nat-int?]])
-(def filter-template
+(mr/def ::filter-template
"A specification for the basic keys in a filter template."
- (mc/schema
- [:map-of
- {:min 1 :max 1}
- [:string {:min 1}]
- filter-value]))
-
-#_(def card-value
- "A specification for the basic keys in the value of a card template."
- (mc/schema
- [:map
- [:dimensions {:optional true} [:vector (mc/schema
- [:map-of
- {:min 1 :max 1}
- [:string {:min 1}]
- [:map
- [:aggregation {:optional true} string?]]])]]
- [:metrics {:optional true} [:vector string?]]
- [:filters {:optional true} [:vector string?]]
- [:card-score {:optional true} nat-int?]]))
-
-#_(def card-template
- "A specification for the basic keys in a card template."
- (mc/schema
- [:map-of
- {:min 1 :max 1}
- [:string {:min 1}]
- card-value]))
-
-#_(def dashboard-template
- "A specification for the basic keys in a dashboard template."
- (mc/schema
- [:map
- [:dimensions {:optional true} [:vector dimension-template]]
- [:metrics {:optional true} [:vector metric-template]]
- [:filters {:optional true} [:vector filter-template]]
- [:cards {:optional true} [:vector card-template]]]))
-
-;; Available values schema -- These are items for which fields have been successfully bound
-
-#_(def available-values
- "Specify the shape of things that are available after dimension to field matching for affinity matching"
- (mc/schema
- [:map
- [:available-dimensions [:map-of [:string {:min 1}] any?]]
- [:available-metrics [:map-of [:string {:min 1}] any?]]
- [:available-filters {:optional true} [:map-of [:string {:min 1}] any?]]]))
-
-;; Schemas for "affinity" functions as these can be particularly confusing
-
-#_(def dimension-set
- "A set of dimensions that belong together. This is the basic unity of affinity."
- [:set string?])
-
-#_(def semantic-affinity-set
- "A set of sematic types that belong together. This is the basic unity of semantic affinity."
- [:set :keyword])
-
-#_(def affinity
- "A collection of things that go together. In this case, we're a bit specialized on
- card affinity, but the key element in the structure is `:base-dims`, which are a
- set of dimensions which, when satisfied, enable this affinity object."
- (mc/schema
- [:map
- [:affinity-name :string]
- [:affinity-set [:set :keyword]]
- [:card-template card-value]
- [:metric-constituent-names [:sequential :string]]
- [:metric-field-types [:set :keyword]]
- [:named-dimensions [:sequential :string]]
- [:score {:optional true} nat-int?]]))
-
-#_(def affinities
- "A sequence of affinity objects."
- (mc/schema
- [:sequential affinity]))
-
-#_(def affinity-old
- "A collection of things that go together. In this case, we're a bit specialized on
- card affinity, but the key element in the structure is `:base-dims`, which are a
- set of dimensions which, when satisfied, enable this affinity object."
- (mc/schema
- [:map
- [:dimensions {:optional true} [:vector string?]]
- [:metrics {:optional true} [:vector string?]]
- [:filters {:optional true} [:vector string?]]
- [:score {:optional true} nat-int?]
- [:affinity-name string?]
- [:base-dims dimension-set]]))
-
-#_(def affinities-old
- "A sequence of affinity objects."
- (mc/schema
- [:sequential affinity-old]))
-
-#_(def affinity-matches
- "A map of named affinities to all dimension sets that are associated with this name."
- (mc/schema
- [:map-of
- :string
- [:vector dimension-set]]))
-
-(def item
+ [:map-of
+ {:min 1 :max 1}
+ [:string {:min 1}]
+ ::filter-value])
+
+(mr/def ::item
"A \"thing\" that we bind to, consisting, generally, of at least a name and id"
- (mc/schema
- [:map
- [:id {:optional true} nat-int?]
- [:name {:optional true} string?]]))
+ [:map
+ [:id {:optional true} nat-int?]
+ [:name {:optional true} string?]])
-(def dim-name->dim-def
+(mr/def ::dim-name->dim-def
"A map of dimension name to dimension definition."
- (mc/schema
- [:map-of :string dimension-value]))
+ [:map-of :string ::dimension-value])
-(def dim-name->matching-fields
+(mr/def ::dim-name->matching-fields
"A map of named dimensions to a map containing the dimension data
and a sequence of matching items satisfying this dimension"
- (mc/schema
- [:map-of :string
- [:map
- [:matches [:sequential item]]]]))
+ [:map-of
+ :string
+ [:map
+ [:matches [:sequential ::item]]]])
-(def dim-name->dim-defs+matches
+(mr/def ::dim-name->dim-defs+matches
"The \"full\" grounded dimensions which matches dimension names
to the dimension definition combined with matching fields."
- (mut/merge
- dim-name->dim-def
- dim-name->matching-fields))
-
-#_(def dimension-map
- "A map of dimension names to item satisfying that dimensions"
- (mc/schema
- [:map-of :string item]))
-
-#_(def dimension-maps
- "A sequence of dimension maps"
- (mc/schema
- [:sequential dimension-map]))
+ [:merge
+ ::dim-name->dim-def
+ ::dim-name->matching-fields])
-(def normalized-metric-template
+(mr/def ::normalized-metric-template
"A \"normalized\" metric template is a map containing the metric name as a key
rather than a map of metric name to the map."
- (mc/schema
- [:map
- [:metric-name :string]
- [:score nat-int?]
- [:metric vector?]]))
+ [:map
+ {:closed true}
+ [:name {:optional true} ::string-or-18n-string]
+ [:metric-name :string]
+ [:score nat-int?]
+ [:metric vector?]])
-(def grounded-metric
+(mr/def ::grounded-metric.definition
+ [:map
+ {:closed true}
+ [:xrays/aggregations {:optional true} [:maybe [:sequential ::aggregation]]]
+ [:xrays/breakouts {:optional true} [:maybe [:sequential ::lib.schema.metadata/column]]]
+ [:xrays/filters {:optional true} [:maybe [:sequential ::external-op]]]])
+
+(mr/def ::grounded-metric
"A metric containing a definition with actual field references/ids rather than dimension references."
- (mc/schema
+ [:map
+ {:closed true}
+ [:metric-name :string]
+ [:metric-title :string]
+ [:metric-score nat-int?]
+ [:metric-definition ::grounded-metric.definition]
+ [:id {:optional true} symbol?]
+ [:position {:optional true} nat-int?]
+ [:dimension-name->field {:optional true} [:map-of :string ::field]]
+ [:card-score {:optional true} number?]
+ [:score-components {:optional true} [:sequential number?]]
+ [:affinity-name {:optional true} :string]
+ [:total-score {:optional true} number?]])
+
+(mr/def ::combined-metric
+ "A grounded metric in which the metric has been augmented with breakouts."
+ [:merge
+ ::grounded-metric
[:map
- [:metric-name :string]
- [:metric-title :string]
- [:metric-score nat-int?]
+ {:closed true}
+ [:group {:optional true} :string]
+ [:card-name {:optional true} :string]
+ [:height {:optional true} number?]
+ [:width {:optional true} number?]
+ [:title {:optional true} :string]
+ [:visualization {:optional true} [:tuple :string :map]]
+ [:metrics {:optional true} [:sequential :string]]
+ [:filters {:optional true} [:sequential :string]]
+ [:description {:optional true} :string]
+ [:dimensions {:optional true} [:sequential [:map-of :string :map]]]
+ ;; HUH??
+ [:order_by {:optional true} [:sequential [:map-of :string [:enum "ascending" "descending"]]]]
+ [:limit {:optional true} pos-int?]
+ [:x_label {:optional true} :string]
[:metric-definition
- [:map
- [:aggregation [:sequential any?]]]]]))
-
-(def combined-metric
- "A grounded metric in which the metric has been augmented with breakouts."
- (mut/merge
- grounded-metric
- (mc/schema
- [:map
- [:metric-definition
+ [:merge
+ ::grounded-metric.definition
[:map
- [:aggregation [:sequential any?]]
- [:breakout [:sequential any?]]]]])))
+ [:xrays/breakouts [:sequential ::lib.schema.metadata/column]]]]]]])
-#_(comment
- (require '[malli.generator :as mg])
- (mg/sample dashboard-template)
- (mg/sample affinities)
- (mg/sample affinity-matches)
- (mg/sample grounded-metric))
+(mr/def ::grounded-filter
+ [:map
+ {:closed true}
+ [:score number?]
+ [:filter ::external-op]
+ [:filter-name :string]])
(mr/def ::field
[:and
@@ -353,3 +272,10 @@
out what the schema is supposed to be yet."
[:map
[:cards {:optional true} [:maybe [:sequential ::card-template]]]])
+
+(mr/def ::grounded-values
+ [:map
+ {:closed true}
+ [:dimensions {:optional true} ::dim-name->matching-fields]
+ [:metrics {:optional true} [:sequential ::grounded-metric]]
+ [:filters {:optional true} [:sequential ::grounded-filter]]])
diff --git a/src/metabase/xrays/automagic_dashboards/util.clj b/src/metabase/xrays/automagic_dashboards/util.clj
index 6b415930ed4..40172b1d800 100644
--- a/src/metabase/xrays/automagic_dashboards/util.clj
+++ b/src/metabase/xrays/automagic_dashboards/util.clj
@@ -4,12 +4,7 @@
[clojure.string :as str]
[medley.core :as m]
[metabase.analyze.core :as analyze]
- ;; legacy usages, do not use legacy MBQL stuff in new code.
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]
- ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.legacy-mbql.util :as mbql.u]
- [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
- [metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.util.match :as lib.util.match]
[metabase.models.interface :as mi]
@@ -49,32 +44,25 @@
[tablespec tables :- [:maybe [:sequential ::ads/source]]]
(filter #(-> % :entity_type (isa? tablespec)) tables))
-(def ^{:arglists '([metric])} saved-metric?
+(defn saved-metric?
"Is metric a saved (V2) metric? (Note that X-Rays do not currently know how to handle Saved V2 Metrics.)"
- ;; legacy usage, do not use legacy MBQL stuff in new code.
- #_{:clj-kondo/ignore [:deprecated-var]}
- (partial mbql.u/is-clause? :metric))
-
-(def ^{:arglists '([metric])} custom-expression?
- "Is this a custom expression?"
- ;; legacy usage, do not use legacy MBQL stuff in new code.
- #_{:clj-kondo/ignore [:deprecated-var]}
- (partial mbql.u/is-clause? :aggregation-options))
+ [metric]
+ (lib/clause-of-type? metric :metric))
(def ^{:arglists '([metric])} adhoc-metric?
"Is this an adhoc metric?"
- (complement (some-fn saved-metric? custom-expression?)))
+ (complement saved-metric?))
(def ^{:arglists '([x])} encode-base64-json
"Encode given object as form-encoded base-64-encoded JSON."
(comp codec/form-encode codec/base64-encode codecs/str->bytes json/encode))
-(mu/defn field-reference->id :- [:maybe [:or ms/NonBlankString ms/PositiveInt]]
+(mu/defn field-reference->id :- [:maybe [:or ms/NonBlankString ::lib.schema.id/field]]
"Extract field ID from a given field reference form."
- [clause]
- (lib.util.match/match-one clause [:field id _] id))
+ [clause :- :mbql.clause/field]
+ (lib.util.match/match-one clause [:field _opts id] id))
-(mu/defn collect-field-references :- [:maybe [:sequential mbql.s/field]]
+(mu/defn collect-field-references :- [:maybe [:sequential :mbql.clause/field]]
"Collect all `:field` references from a given form."
[form]
(lib.util.match/match form :field &match))
@@ -87,7 +75,8 @@
field-id-or-name-or-clause :- [:or
::lib.schema.id/field
ms/NonBlankString
- ::mbql.s/field-or-expression-ref]]
+ :mbql.clause/field
+ :mbql.clause/expression]]
(let [id-or-name (if (sequential? field-id-or-name-or-clause)
(field-reference->id field-id-or-name-or-clause)
field-id-or-name-or-clause)]
@@ -114,28 +103,3 @@
"Generate a parameter ID for the given field. In X-ray dashboards a parameter is mapped to a single field only."
[field]
(-> field ((juxt :id :name :unit)) hash str))
-
-(defn do-with-legacy-query
- "Call
-
- (apply f query args)
-
- with `query` converted to a legacy MBQL query if needed."
- [query f & args]
- (when (seq query)
- (case (lib/normalized-mbql-version query)
- :mbql-version/legacy (apply f query args)
- :mbql-version/mbql5 (apply f #_{:clj-kondo/ignore [:discouraged-var]} (lib/->legacy-MBQL query) args))))
-
-(defn do-with-mbql5-query
- "Call
-
- (apply f query args)
-
- with `query` converted to an MBQL 5 query if needed."
- [query f & args]
- (when (seq query)
- (case (lib/normalized-mbql-version query)
- :mbql-version/legacy (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]
- (apply f (lib-be/normalize-query query) args))
- :mbql-version/mbql5 (apply f query args))))
diff --git a/src/metabase/xrays/related.clj b/src/metabase/xrays/related.clj
index b408e13ac00..c98b91cb868 100644
--- a/src/metabase/xrays/related.clj
+++ b/src/metabase/xrays/related.clj
@@ -4,12 +4,14 @@
[clojure.set :as set]
[medley.core :as m]
[metabase.api.common :as api]
- ;; legacy usage, do not use legacy MBQL stuff in new code.
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]
+ [metabase.lib.core :as lib]
+ [metabase.lib.schema.util :as lib.schema.util]
[metabase.models.interface :as mi]
[metabase.query-processor.util :as qp.util]
+ [metabase.segments.schema :as segments.schema]
+ [metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
- [metabase.xrays.automagic-dashboards.util :as magic.util]
+ [metabase.xrays.automagic-dashboards.schema :as ads]
[toucan2.core :as t2]))
(def ^:private ^Long max-best-matches 3)
@@ -17,24 +19,22 @@
(def ^:private ^Long max-matches (+ max-best-matches
max-serendipity-matches))
-(def ^:private ContextBearingForm
+(mr/def ::context-bearing-form
[:cat
[:and
[:or :string :keyword]
[:fn
{:error/message "field, metric, or segment"}
(comp #{:field :metric :segment}
- qp.util/normalize-token)]]
+ keyword)]]
[:* :any]])
(defn- collect-context-bearing-forms
[form]
- ;; legacy usage, temporary until we convert X-Rays to Lib
- (let [form #_{:clj-kondo/ignore [:deprecated-var]} (mbql.normalize/normalize-fragment [:query :filter] form)]
- (into #{}
- (comp (filter (mr/validator ContextBearingForm))
- (map #(update % 0 qp.util/normalize-token)))
- (tree-seq sequential? identity form))))
+ (into #{}
+ (comp (filter (mr/validator ::context-bearing-form))
+ (map #(update % 0 qp.util/normalize-token)))
+ (tree-seq sequential? identity form)))
(defmulti definition
"Return the relevant parts of a given entity's definition. Relevant parts are those that carry semantic meaning, and
@@ -42,23 +42,23 @@
{:arglists '([instance])}
mi/model)
-(defmethod definition :xrays/Metric
- [metric]
- (-> metric :definition ((juxt :aggregation :filter))))
+(mu/defmethod definition :xrays/Metric
+ [metric :- ::ads/metric]
+ (-> metric :xrays/aggregation))
(defmethod definition :model/Card
[card]
(-> card
:dataset_query
- (magic.util/do-with-legacy-query #(-> % :query ((juxt :breakout :aggregation :expressions :fields))))))
+ ((juxt lib/breakouts lib/aggregations lib/expressions lib/fields))))
-(defmethod definition :model/Segment
- [segment]
- (-> segment :definition :filter))
+(mu/defmethod definition :model/Segment
+ [segment :- ::segments.schema/segment]
+ (-> segment :definition :filter lib/->pMBQL))
(defmethod definition :model/Field
[field]
- [[:field-id (:id field)]])
+ [[:field {:lib/uuid (str (random-uuid))} (:id field)]])
(defn- similarity
"How similar are entities `a` and `b` based on a structural comparison of their
@@ -69,8 +69,8 @@
(more context-bearing forms), so we just check if the less specifc form is a
subset of the more specific one."
[a b]
- (let [context-a (-> a definition collect-context-bearing-forms)
- context-b (-> b definition collect-context-bearing-forms)
+ (let [context-a (-> a definition collect-context-bearing-forms lib.schema.util/remove-lib-uuids)
+ context-b (-> b definition collect-context-bearing-forms lib.schema.util/remove-lib-uuids)
overlap (set/intersection context-a context-b)
min-overlap (min (count context-a) (count context-b))]
(/ (count overlap)
diff --git a/src/metabase/xrays/transforms/materialize.clj b/src/metabase/xrays/transforms/materialize.clj
index 171f411e154..73e932b282d 100644
--- a/src/metabase/xrays/transforms/materialize.clj
+++ b/src/metabase/xrays/transforms/materialize.clj
@@ -3,6 +3,7 @@
[metabase.api.common :as api]
[metabase.collections.models.collection :as collection]
[metabase.lib-be.core :as lib-be]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.queries.core :as queries]
[metabase.queries.schema :as queries.schema]
[metabase.query-processor.preprocess :as qp.preprocess]
@@ -18,12 +19,13 @@
(t2/select-one [:model/Collection :location :id]
:id (get-or-create-root-container-collection!))))
-(defn get-collection
+(mu/defn get-collection :- [:maybe ::lib.schema.id/collection]
"Get collection named `collection-name`. If no location is given root collection for automatically
generated transforms is assumed (see `get-or-create-root-container-collection!`)."
([collection-name]
(get-collection collection-name (root-container-location)))
- ([collection-name location]
+ ([collection-name :- :string
+ location :- :string]
(t2/select-one-pk :model/Collection
:name collection-name
:location location)))
diff --git a/test/metabase/parameters/schema_test.cljc b/test/metabase/parameters/schema_test.cljc
index 27c0e7ffe5c..5fa8c3f48a6 100644
--- a/test/metabase/parameters/schema_test.cljc
+++ b/test/metabase/parameters/schema_test.cljc
@@ -18,6 +18,6 @@
(parameters.schema/normalize-parameter-mapping
{"parameter_id" "53bb6214", "card_id" -1, "target" ["variable" ["template-tag" "id"]]})))))
-(deftest ^:paralell normalize-parameters-without-adding-default-type-test
+(deftest ^:parallel normalize-parameters-without-adding-default-type-test
(is (= [{:id "x", :target [:dimension [:template-tag "y"]]}]
(parameters.schema/normalize-parameters-without-adding-default-types [{"id" "x", "target" ["dimension" ["template-tag" "y"]]}]))))
diff --git a/test/metabase/xrays/api/automagic_dashboards_test.clj b/test/metabase/xrays/api/automagic_dashboards_test.clj
index 0904983667e..8a89ae8f19b 100644
--- a/test/metabase/xrays/api/automagic_dashboards_test.clj
+++ b/test/metabase/xrays/api/automagic_dashboards_test.clj
@@ -238,12 +238,10 @@
(is (some?
(api-call! "table/%s/compare/segment/%s"
[(mt/id :venues) segment-id]))))
-
(testing "GET /api/automagic-dashboards/table/:id/rule/example/indepth/compare/segment/:segment-id"
(is (some?
(api-call! "table/%s/rule/example/indepth/compare/segment/%s"
[(mt/id :venues) segment-id]))))
-
(testing "GET /api/automagic-dashboards/adhoc/:id/cell/:cell-query/compare/segment/:segment-id"
(is (some?
(api-call! "adhoc/%s/cell/%s/compare/segment/%s"
@@ -611,7 +609,7 @@
{:id (mt/id :categories)}])}
(mt/user-http-request :crowberto :get 200 (str "automagic-dashboards/table/" (mt/id :venues) "/query_metadata")))))
-(deftest ^:paralell allow-equals-in-base-64-encoding-for-now-test
+(deftest ^:parallel allow-equals-in-base-64-encoding-for-now-test
(testing "Allow `=` in the route part of X-Rays API requests... for now"
(doseq [schema [::api.magic/base-64-encoded-json
::api.magic/entity-id-or-query]]
diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.clj
index 811fdd32bda..5cf71b585a8 100644
--- a/test/metabase/xrays/automagic_dashboards/core_test.clj
+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj
@@ -9,6 +9,7 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema.id :as lib.schema.id]
+ [metabase.lib.test-metadata :as meta]
[metabase.lib.util.match :as lib.util.match]
[metabase.models.interface :as mi]
[metabase.permissions.models.permissions :as perms]
@@ -26,15 +27,18 @@
[metabase.xrays.automagic-dashboards.core :as magic]
[metabase.xrays.automagic-dashboards.dashboard-templates :as dashboard-templates]
[metabase.xrays.automagic-dashboards.interesting :as interesting]
- [metabase.xrays.automagic-dashboards.util :as magic.util]
[metabase.xrays.test-util.automagic-dashboards :as automagic-dashboards.test]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
+(deftest ^:parallel normalize-query-filter-test
+ (is (=? {:query-filter [[:= {} [:field {} 1] 2]]}
+ (lib/normalize ::magic/automagic-analysis.opts {:query-filter [:= [:field 1 nil] 2]}))))
+
;;; ------------------- Dashboard template matching -------------------
-(deftest dashboard-template-matching-test
+(deftest ^:parallel dashboard-template-matching-test
(is (= [:entity/UserTable :entity/GenericTable :entity/*]
(->> (mt/id :users)
(t2/select-one :model/Table :id)
@@ -42,7 +46,7 @@
(#'magic/matching-dashboard-templates (dashboard-templates/get-dashboard-templates ["table"]))
(map (comp first :applies_to))))))
-(deftest dashboard-template-matching-test-2
+(deftest ^:parallel dashboard-template-matching-test-2
(testing "Test fallback to GenericTable"
(is (= [:entity/GenericTable :entity/*]
(->> (-> (t2/select-one :model/Table :id (mt/id :users))
@@ -53,7 +57,7 @@
;;; ------------------- `->root source` -------------------
-(deftest source-root-table-test
+(deftest ^:parallel source-root-table-test
(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Table"
(mt/dataset test-data
(testing "The source of a table is the table itself"
@@ -63,7 +67,7 @@
(is (= entity table))
(is (= source entity)))))))
-(deftest source-root-field-test
+(deftest ^:parallel source-root-field-test
(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Field"
(mt/dataset test-data
(testing "The source of a field is the originating table of the field"
@@ -73,7 +77,7 @@
(is (= source table))
(is (= entity field)))))))
-(deftest source-root-card-test
+(deftest ^:parallel source-root-card-test
(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Card"
(mt/dataset test-data
(testing "Card sourcing has four branches..."
@@ -92,7 +96,7 @@
(is (=? (dissoc source :dataset_query)
(assoc card :entity_type :entity/GenericTable))))))))))
-(deftest source-root-card-test-2
+(deftest ^:parallel source-root-card-test-2
(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Card"
(mt/dataset test-data
(testing "Card sourcing has four branches..."
@@ -118,7 +122,7 @@
(is (=? (dissoc source :entity_type)
nested-query)))))))))
-(deftest source-root-card-test-3
+(deftest ^:parallel source-root-card-test-3
(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Card"
(mt/dataset test-data
(testing "Card sourcing has four branches..."
@@ -134,7 +138,7 @@
(is (=? (dissoc source :dataset_query :entity_type)
card))))))))))
-(deftest source-root-card-test-4
+(deftest ^:parallel source-root-card-test-4
(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Card"
(mt/dataset test-data
(testing "Card sourcing has four branches..."
@@ -155,7 +159,7 @@
card))
(is (= source (t2/select-one :model/Table :id table-id))))))))))
-(deftest source-root-query-test
+(deftest ^:parallel source-root-query-test
(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Query"
(mt/dataset test-data
(testing "The source of a query is the underlying datasource of the query"
@@ -163,15 +167,17 @@
:model/Query
{:database-id (mt/id)
:table-id (mt/id :orders)
- :dataset_query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :orders)
- :aggregation [[:count]]}}})
+ :dataset_query (lib/query
+ (mt/metadata-provider)
+ {:database (mt/id)
+ :type :query
+ :query {:source-table (mt/id :orders)
+ :aggregation [[:count]]}})})
{:keys [entity source]} (#'magic/->root query)]
(is (= entity query))
(is (= source (t2/select-one :model/Table (mt/id :orders)))))))))
-(deftest source-root-segment-test
+(deftest ^:parallel source-root-segment-test
(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Segment"
(testing "The source of a segment is its underlying table."
(mt/with-temp [:model/Segment segment {:table_id (mt/id :venues)
@@ -198,7 +204,7 @@
;; These test names were named by staring at them for a while, so they may be misleading
-(deftest automagic-analysis-test
+(deftest ^:parallel automagic-analysis-test
(mt/with-test-user :rasta
(automagic-dashboards.test/with-rollback-only-transaction
(doseq [[table cardinality] (map vector
@@ -206,7 +212,7 @@
[2 8 11 11 15 17 5 7])]
(test-automagic-analysis table cardinality)))))
-(deftest automagic-analysis-test-2
+(deftest ^:parallel automagic-analysis-test-2
(mt/with-test-user :rasta
(automagic-dashboards.test/with-rollback-only-transaction
(is (= 1
@@ -215,7 +221,7 @@
(filter :card)
count))))))
-(deftest weird-characters-in-names-test
+(deftest ^:parallel weird-characters-in-names-test
(mt/with-test-user :rasta
(automagic-dashboards.test/with-rollback-only-transaction
(-> (t2/select-one :model/Table :id (mt/id :venues))
@@ -224,7 +230,7 @@
;; Cardinality of cards genned from fields is much more labile than anything else
;; Not just with respect to drivers, but all sorts of other stuff that makes it chaotic
-(deftest mass-field-test
+(deftest ^:parallel mass-field-test
(mt/with-test-user :rasta
(automagic-dashboards.test/with-rollback-only-transaction
(doseq [field (t2/select :model/Field
@@ -233,7 +239,7 @@
{:order-by [[:id :asc]]})]
(is (pos? (count (:dashcards (magic/automagic-analysis field {})))))))))
-(deftest parameter-mapping-test
+(deftest ^:parallel parameter-mapping-test
(mt/dataset test-data
(testing "mbql queries have parameter mappings with field ids"
(let [table (t2/select-one :model/Table :id (mt/id :products))
@@ -246,7 +252,7 @@
(:dashcards dashboard))]
(is (= expected-targets actual-targets))))))
-(deftest parameter-mapping-test-2
+(deftest ^:parallel parameter-mapping-test-2
(mt/dataset test-data
(testing "native queries have parameter mappings with field ids"
(let [query (mt/native-query {:query "select * from products"})]
@@ -607,7 +613,7 @@
(for [dashcard (:dashcards dashboard)
:let [query (get-in dashcard [:card :dataset_query])]
:when query
- :let [breakouts (magic.util/do-with-mbql5-query query lib/breakouts)]
+ :let [breakouts (lib/breakouts query)]
id (lib.util.match/match breakouts
[:field (_opts :guard :binning) (id :guard pos-int?)]
id)]
@@ -641,7 +647,7 @@
temporal-field-ids (for [dashcard (:dashcards dashboard)
:let [query (get-in dashcard [:card :dataset_query])]
:when query
- :let [breakouts (magic.util/do-with-mbql5-query query lib/breakouts)]
+ :let [breakouts (lib/breakouts query)]
id (lib.util.match/match breakouts
[:field (_opts :guard :temporal-unit) (id :guard pos-int?)]
id)]
@@ -756,7 +762,7 @@
(is (false? (str/ends-with? question-dashboard-name "model")))
(is (true? (str/ends-with? question-dashboard-name (format "\"%s\"" (:name question-card))))))))))))
-(deftest model-based-automagic-dashboards-have-correct-parameter-mappings-test
+(deftest ^:parallel model-based-automagic-dashboards-have-correct-parameter-mappings-test
(testing "Dashcard parameter mappings have valid targets when X-raying models (#58214)"
(mt/dataset test-data
(mt/with-temp
@@ -907,14 +913,29 @@
(mt/with-test-user :rasta
(automagic-dashboards.test/with-rollback-only-transaction
(perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id) =
- (let [all-dashcard-filters (->> (magic/automagic-analysis entity {:cell-query cell-query :show :all})
- :dashcards
- (keep #(-> %
- (get-in [:card :dataset_query])
- (magic.util/do-with-legacy-query get-in [:query :filter]))))
- filter-contains-cell-query? #(= cell-query (some #{cell-query} %))]
- (is (pos? (count all-dashcard-filters)))
- (is (every? filter-contains-cell-query? all-dashcard-filters))))))))))
+ (let [dashcards (:dashcards (magic/automagic-analysis entity {:cell-query cell-query :show :all}))]
+ (is (= ["# Summary"
+ "Total Venues"
+ "Distinct Category ID"
+ "# Where these Venues are"
+ "Venues by coordinates"
+ "# How these Venues are distributed"
+ "Venues per Price"]
+ (map (some-fn (comp :name :card) (comp :text :visualization_settings)) dashcards)))
+ (let [all-dashcard-filters (keep #(some-> %
+ (get-in [:card :dataset_query])
+ not-empty
+ lib/filters)
+ dashcards)]
+ (is (=? [[[:> {} [:field {:base-type :type/Integer} (mt/id :venues :price)] 10]
+ [:= {} [:field {} (mt/id :venues :category_id)] 2]]
+ [[:> {} [:field {:base-type :type/Integer} (mt/id :venues :price)] 10]
+ [:= {} [:field {} (mt/id :venues :category_id)] 2]]
+ [[:> {} [:field {:base-type :type/Integer} (mt/id :venues :price)] 10]
+ [:= {} [:field {} (mt/id :venues :category_id)] 2]]
+ [[:> {} [:field {:base-type :type/Integer} (mt/id :venues :price)] 10]
+ [:= {} [:field {} (mt/id :venues :category_id)] 2]]]
+ all-dashcard-filters)))))))))))
(deftest complicated-card-cell-test
(mt/with-non-admin-groups-no-root-collection-perms
@@ -931,7 +952,7 @@
(-> (t2/select-one :model/Card :id card-id)
(test-automagic-analysis [:= [:field (mt/id :venues :category_id) nil] 2] 7)))))))
-(deftest adhoc-filter-test
+(deftest ^:parallel adhoc-filter-test
(mt/with-test-user :rasta
(automagic-dashboards.test/with-rollback-only-transaction
(let [q (api.automagic-dashboards/adhoc-query-instance {:query {:filter [:> [:field (mt/id :venues :price) nil] 10]
@@ -940,7 +961,7 @@
:database (mt/id)})]
(test-automagic-analysis q 7)))))
-(deftest adhoc-count-test
+(deftest ^:parallel adhoc-count-test
(mt/with-test-user :rasta
(automagic-dashboards.test/with-rollback-only-transaction
(let [q (api.automagic-dashboards/adhoc-query-instance {:query {:aggregation [[:count]]
@@ -950,7 +971,7 @@
:database (mt/id)})]
(test-automagic-analysis q 17)))))
-(deftest adhoc-fk-breakout-test
+(deftest ^:parallel adhoc-fk-breakout-test
(mt/with-test-user :rasta
(automagic-dashboards.test/with-rollback-only-transaction
(let [q (api.automagic-dashboards/adhoc-query-instance {:query {:aggregation [[:count]]
@@ -960,7 +981,7 @@
:database (mt/id)})]
(test-automagic-analysis q 9)))))
-(deftest adhoc-filter-cell-test
+(deftest ^:parallel adhoc-filter-cell-test
(mt/with-test-user :rasta
(automagic-dashboards.test/with-rollback-only-transaction
(let [q (api.automagic-dashboards/adhoc-query-instance {:query {:filter [:> [:field (mt/id :venues :price) nil] 10]
@@ -969,22 +990,27 @@
:database (mt/id)})]
(test-automagic-analysis q [:= [:field (mt/id :venues :category_id) nil] 2] 7)))))
-(deftest join-splicing-test
+(deftest ^:parallel join-splicing-test
(mt/with-test-user :rasta
(automagic-dashboards.test/with-rollback-only-transaction
(let [join-vec [{:source-table (mt/id :categories)
:condition [:= [:field (mt/id :categories :id) nil] 1]
:strategy :left-join
:alias "Dealios"}]
- q (api.automagic-dashboards/adhoc-query-instance {:query {:source-table (mt/id :venues)
- :joins join-vec
- :aggregation [[:sum [:field (mt/id :categories :id) {:join-alias "Dealios"}]]]}
- :type :query
+ q (api.automagic-dashboards/adhoc-query-instance {:query {:source-table (mt/id :venues)
+ :joins join-vec
+ :aggregation [[:sum [:field (mt/id :categories :id) {:join-alias "Dealios"}]]]}
+ :type :query
:database (mt/id)})
res (magic/automagic-analysis q {})
cards (vec (:dashcards res))
- join-member (get-in cards [2 :card :dataset_query :query :joins])]
- (is (= join-vec join-member))))))
+ join-member (lib/joins (get-in cards [2 :card :dataset_query]))]
+ (is (=? [{:strategy :left-join
+ :alias "Dealios"
+ :conditions [[:= {} [:field {} (mt/id :categories :id)] 1]]
+ :lib/type :mbql/join
+ :stages [{:lib/type :mbql.stage/mbql, :source-table (mt/id :categories)}]}]
+ join-member))))))
;;; ------------------- /candidates -------------------
@@ -999,7 +1025,7 @@
[:* :any]]
(magic/candidate-tables (mt/db)))))))))
-(deftest candidates-test-2
+(deftest ^:parallel candidates-test-2
(testing "/candidates"
(testing "should work with unanalyzed tables"
(mt/with-test-user :rasta
@@ -1010,7 +1036,7 @@
(is (=? [{:tables [{:table {:id table-id}}]}]
(magic/candidate-tables (t2/select-one :model/Database :id db-id)))))))))
-(deftest call-count-test
+(deftest ^:parallel call-count-test
(mt/with-temp [:model/Database {db-id :id} {}
:model/Table {table-id :id} {:db_id db-id}
:model/Field _ {:table_id table-id}
@@ -1022,7 +1048,7 @@
(magic/candidate-tables database)
(is (= 2 (call-count)))))))))
-(deftest empty-table-test
+(deftest ^:parallel empty-table-test
(testing "candidate-tables should work with an empty Table (no Fields)"
(mt/with-temp [:model/Database db {}
:model/Table _ {:db_id (:id db)}]
@@ -1030,7 +1056,7 @@
(is (= []
(magic/candidate-tables db)))))))
-(deftest enhance-table-stats-test
+(deftest ^:parallel enhance-table-stats-test
(mt/with-temp [:model/Database {db-id :id} {}
:model/Table {table-id :id} {:db_id db-id}
:model/Field _ {:table_id table-id :semantic_type :type/PK}
@@ -1042,7 +1068,7 @@
(-> (#'magic/load-tables-with-enhanced-table-stats [[:= :id table-id]])
first)))))))
-(deftest enhance-table-stats-fk-test
+(deftest ^:parallel enhance-table-stats-fk-test
(mt/with-temp [:model/Database {db-id :id} {}
:model/Table {table-id :id} {:db_id db-id}
:model/Field _ {:table_id table-id :semantic_type :type/PK}
@@ -1056,7 +1082,7 @@
;;; ------------------- Definition overloading -------------------
-(deftest most-specific-definition-test
+(deftest ^:parallel most-specific-definition-test
(testing "Identity"
(is (= :d1
(-> [{:d1 {:field_type [:type/Category] :score 100}}]
@@ -1064,7 +1090,7 @@
first
key)))))
-(deftest ancestors-definition-test
+(deftest ^:parallel ancestors-definition-test
(testing "Base case: more ancestors"
(is (= :d2
(-> [{:d1 {:field_type [:type/Category] :score 100}}
@@ -1073,7 +1099,7 @@
first
key)))))
-(deftest definition-tiebreak-test
+(deftest ^:parallel definition-tiebreak-test
(testing "Break ties based on the number of additional filters"
(is (= :d3
(-> [{:d1 {:field_type [:type/Category] :score 100}}
@@ -1085,7 +1111,7 @@
first
key)))))
-(deftest definition-tiebreak-score-test
+(deftest ^:parallel definition-tiebreak-score-test
(testing "Break ties on score"
(is (= :d2
(-> [{:d1 {:field_type [:type/Category] :score 100}}
@@ -1095,7 +1121,7 @@
first
key)))))
-(deftest definition-tiebreak-precedence-test
+(deftest ^:parallel definition-tiebreak-precedence-test
(testing "Number of additional filters has precedence over score"
(is (= :d3
(-> [{:d1 {:field_type [:type/Category] :score 100}}
@@ -1109,7 +1135,7 @@
;;; -------------------- Filters --------------------
-(deftest most-specific-definition-inner-shape-test
+(deftest ^:parallel most-specific-definition-inner-shape-test
(testing "Ensure we have examples to understand the shape returned from most-specific-definition"
(mt/dataset test-data
(testing ""
@@ -1179,7 +1205,7 @@
candidate-bindings
(#'interesting/most-specific-matched-dimension))))))))))))
-(deftest bind-dimensions-inner-shape-test
+(deftest ^:parallel bind-dimensions-inner-shape-test
(testing "Ensure we have examples to understand the shape returned from bind-dimensions"
(mt/dataset test-data
(testing "Clearly demonstrate the mechanism of full dimension binding"
@@ -1212,7 +1238,7 @@
:score 60}}
(update-in bound-dimensions ["Loc" :matches] (partial sort-by :id))))))))))
-(deftest binding-functions-with-all-same-names-and-types-test
+(deftest ^:parallel binding-functions-with-all-same-names-and-types-test
(testing "Ensure expected behavior when multiple columns alias to the same base column and display metadata uses the
same name for all columns."
(mt/dataset test-data
@@ -1279,7 +1305,7 @@
;;; -------------------- Ensure generation of subcards via related (includes indepth, drilldown) --------------------
-(deftest related-card-generation-test
+(deftest ^:parallel related-card-generation-test
(testing "Ensure that the `related` function is called and the right cards are created."
(mt/with-test-user :rasta
(mt/dataset test-data
@@ -1313,17 +1339,18 @@
(update :zoom-in (comp vec (partial sort-by :title)))
(update :related (comp vec (partial sort-by :title)))))))))))
-(deftest singular-cell-dimensions-test
+(deftest ^:parallel singular-cell-dimensions-test
(testing "Find the cell dimensions for a cell query"
(is (= #{1 2 "TOTAL"}
(#'magic/singular-cell-dimension-field-ids
- {:cell-query
- [:and
- [:= [:field 1 nil]]
- [:= [:field 2 nil]]
- [:= [:field "TOTAL" {:base-type :type/Number}]]]})))))
-
-(deftest adhoc-query-with-explicit-joins-14793-test
+ {:database 1
+ :cell-query (lib/normalize
+ [:and
+ [:= [:field 1 nil] 5]
+ [:= [:field 2 nil] 6]
+ [:= [:field "TOTAL" {:base-type :type/Number}] 7]])})))))
+
+(deftest ^:parallel adhoc-query-with-explicit-joins-14793-test
(testing "A verification of the fix for https://github.com/metabase/metabase/issues/14793,
X-rays fails on explicit joins, when metric is for the joined table"
(mt/dataset test-data
@@ -1360,7 +1387,7 @@
(testing "An expectation check -- this dashboard should produce this group heading"
(is (contains? section-headings expected-section)))))))))
-(deftest compare-to-the-rest-25278+32557-test
+(deftest ^:parallel compare-to-the-rest-25278+32557-test
(testing "Ensure valid queries are generated for an automatic comparison dashboard (fixes 25278 & 32557)"
(mt/dataset test-data
(mt/with-test-user :crowberto
@@ -1390,12 +1417,16 @@
(is (some? card-with-cell-query))
(is (some? card-without-cell-query)))
(testing "The cell-query exists in only one of the cards"
- (is (not= cell-query (get-in base-query [:query :filter])))
- (is (= cell-query (get-in filtered-query [:query :filter]))))
+ (is (nil? (get-in base-query [:query :filter])))
+ (is (=? [[:=
+ {}
+ [:field {:base-type :type/Text, :join-alias "Products"} (mt/id :products :title)]
+ "Intelligent Granite Hat"]]
+ (lib/filters filtered-query))))
(testing "Join aliases exist in the queries"
;; The reason issues 25278 & 32557 would blow up is the lack of join aliases.
- (is (= ["Products"] (map :alias (get-in base-query [:query :joins]))))
- (is (= ["Products"] (map :alias (get-in filtered-query [:query :joins])))))
+ (is (= ["Products"] (map :alias (lib/joins base-query))))
+ (is (= ["Products"] (map :alias (lib/joins filtered-query)))))
(testing "Card queries are both executable and produce different results"
;; Note that issues 25278 & 32557 would blow up on the filtered queries.
(let [base-data (get-in (qp/process-query base-query) [:data :rows])
@@ -1404,7 +1435,7 @@
(is (some? filtered-data))
(is (not= base-data filtered-data)))))))))
-(deftest compare-to-the-rest-with-expression-16680-test
+(deftest ^:parallel compare-to-the-rest-with-expression-16680-test
(testing "Ensure a valid comparison dashboard is generated with custom expressions (fixes 16680)"
(mt/dataset test-data
(mt/with-test-user :crowberto
@@ -1437,7 +1468,7 @@
series-card-label "Number of Orders per day of the week (Number of Orders where TestColumn is 2 and Created At is in February 2019)"
{[{series-dataset-query :dataset_query}] :series
- {card-dataset-query :dataset_query} :card
+ {card-dataset-query :dataset_query} :card
:as series-card} (some
(fn [{{card-name :name} :card :as dashcard}]
(when (= series-card-label card-name)
@@ -1448,11 +1479,16 @@
(is (some? card-with-cell-query))
(is (some? card-without-cell-query)))
(testing "The cell-query exists in only one of the cards"
- (is (not= cell-query (get-in base-query [:query :filter])))
- (is (= cell-query (get-in filtered-query [:query :filter]))))
+ (is (empty? (lib/filters base-query)))
+ (is (=? [[:and {}
+ [:= {} [:expression {} "TestColumn"] 2]
+ [:= {}
+ [:field {:temporal-unit :month} (mt/id :orders :created_at)]
+ "2019-02-01T00:00:00Z"]]]
+ (lib/filters filtered-query))))
(testing "Expressions exist in the queries"
- (is (= {"TestColumn" [:+ 1 1]} (get-in base-query [:query :expressions])))
- (is (= {"TestColumn" [:+ 1 1]} (get-in filtered-query [:query :expressions]))))
+ (is (=? [[:+ {:lib/expression-name "TestColumn"} 1 1]] (lib/expressions base-query)))
+ (is (=? [[:+ {:lib/expression-name "TestColumn"} 1 1]] (lib/expressions filtered-query))))
(testing "Card queries are both executable and produce different results"
(let [base-data (get-in (qp/process-query base-query) [:data :rows])
filtered-data (get-in (qp/process-query filtered-query) [:data :rows])]
@@ -1470,42 +1506,48 @@
(is (some? card-dataset-query))
(is (= 7 (:row_count (qp/process-query card-dataset-query)))))))))))
-(deftest preserve-entity-element-test
+(deftest ^:parallel preserve-entity-element-test
(testing "Join preservation scenarios: merge, empty expressions, no expressions, no card"
- (is (= [[{:strategy :left-join, :alias "Orders"}
- {:strategy :left-join, :alias "Products"}]
- [{:strategy :left-join, :alias "Products"}]
- [{:strategy :left-join, :alias "Products"}]
+ (is (= [[{:alias "Orders"}
+ {:alias "Products"}]
+ [{:alias "Products"}]
+ [{:alias "Products"}]
nil]
(->>
- (#'magic/preserve-entity-element
- {:dashcards [{:card {:dataset_query {:database 1, :type :query, :query {:joins [{:strategy :left-join :alias "Orders"}]}}}}
- {:card {:dataset_query {:database 1, :type :query, :query {:joins []}}}}
- {:card {:dataset_query {:database 1, :type :query, :query {}}}}
+ (#'magic/preserve-joins
+ {:dashcards [{:card {:dataset_query (-> (lib/query meta/metadata-provider (meta/table-metadata :reviews))
+ (lib/join (-> (lib/join-clause (meta/table-metadata :orders))
+ (lib/with-join-conditions [(lib/= (meta/field-metadata :reviews :id)
+ (meta/field-metadata :orders :id))]))))}}
+ {:card {:dataset_query (lib/query meta/metadata-provider (meta/table-metadata :reviews))}}
+ {:card {:dataset_query (lib/query meta/metadata-provider (meta/table-metadata :reviews))}}
{:viz_settings nil}]}
- {:dataset_query {:database 1, :type :query, :query {:joins [{:strategy :left-join :alias "Products"}]}}}
- :joins)
+ {:dataset_query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/join (meta/table-metadata :products)))})
:dashcards
- (mapv (comp :joins :query :dataset_query :card)))))))
+ (mapv (fn [dashcard]
+ (some-> dashcard :card :dataset_query not-empty lib/joins (->> (map #(select-keys % [:strategy :alias])))))))))))
-(deftest preserve-entity-element-test-2
+(deftest ^:parallel preserve-entity-element-test-2
(testing "Expression preservation scenarios: merge, empty expressions, no expressions, no card"
- (is (= [{"Existing" [:- 1 1] "TestColumn" [:+ 1 1]}
- {"TestColumn" [:+ 1 1]}
- {"TestColumn" [:+ 1 1]}
- nil]
- (->>
- (#'magic/preserve-entity-element
- {:dashcards [{:card {:dataset_query {:database 1, :type :query, :query {:expressions {"Existing" [:- 1 1]}}}}}
- {:card {:dataset_query {:database 1, :type :query, :query {:expressions {}}}}}
- {:card {:dataset_query {:database 1, :type :query, :query {}}}}
- {:viz_settings nil}]}
- {:dataset_query {:database 1, :type :query, :query {:expressions {"TestColumn" [:+ 1 1]}}}}
- :expressions)
- :dashcards
- (mapv (comp :expressions :query :dataset_query :card)))))))
-
-(deftest compare-to-the-rest-15655-test
+ (is (=? [[[:- {:lib/expression-name "Existing"} 1 1]
+ [:+ {:lib/expression-name "TestColumn"} 1 1]]
+ [[:+ {:lib/expression-name "TestColumn"} 1 1]]
+ [[:+ {:lib/expression-name "TestColumn"} 1 1]]
+ nil]
+ (->>
+ (#'magic/preserve-expressions
+ {:dashcards [{:card {:dataset_query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/expression "Existing" (lib/- 1 1)))}}
+ {:card {:dataset_query (lib/query meta/metadata-provider (meta/table-metadata :venues))}}
+ {:card {:dataset_query (lib/query meta/metadata-provider (meta/table-metadata :venues))}}
+ {:viz_settings nil}]}
+ {:dataset_query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/expression "TestColumn" (lib/+ 1 1)))})
+ :dashcards
+ (mapv #(some-> % :card :dataset_query not-empty lib/expressions)))))))
+
+(deftest ^:parallel compare-to-the-rest-15655-test
(testing "Questions based on native questions should produce a valid dashboard. (#15655)"
(let [native-query {:native {:query "select * from people limit 1"}
:type :native
@@ -1570,7 +1612,7 @@
;;; this is a fixed version of the test above that correctly generates the second card so that it matches the equivalent
;;; Cypress test spec.
-(deftest compare-to-the-rest-15655-fixed-test
+(deftest ^:parallel compare-to-the-rest-15655-fixed-test
(testing "Questions based on native questions should produce a valid dashboard. (#15655)"
(mt/with-temp [:model/Card {native-card-id :id} (merge (mt/card-with-source-metadata-for-query
(lib/native-query (mt/metadata-provider) "select * from people LIMIT 100;"))
@@ -1607,7 +1649,7 @@
;; and "cell-query" base-64 encoded
(:dashcards (magic/automagic-analysis card {:cell-query cell-query})))))))))
-(deftest source-fields-are-populated-for-aggregations-38618-test
+(deftest ^:parallel source-fields-are-populated-for-aggregations-38618-test
(testing "X-ray aggregates (metrics) with source fields in external tables should properly fill in `:source-field` (#38618)"
(mt/dataset test-data
(let [dashcard (->> (magic/automagic-analysis (t2/select-one :model/Table :id (mt/id :reviews)) {:show :all})
@@ -1618,9 +1660,11 @@
first)
aggregate (-> dashcard
(get-in [:card :dataset_query])
- (magic.util/do-with-legacy-query get-in [:query :aggregation]))]
+ lib/aggregations)]
(testing "Fields requiring a join should have :source-field populated in the aggregate."
- (is (= [[:distinct [:field (mt/id :products :id)
- ;; This should be present vs. nil (value before issue)
- {:source-field (mt/id :reviews :product_id)}]]]
- aggregate)))))))
+ (is (=? [[:distinct {}
+ [:field
+ ;; This should be present vs. nil (value before issue)
+ {:source-field (mt/id :reviews :product_id)}
+ (mt/id :products :id)]]]
+ aggregate)))))))
diff --git a/test/metabase/xrays/automagic_dashboards/filters_test.clj b/test/metabase/xrays/automagic_dashboards/filters_test.clj
index 420bd405396..177606f7750 100644
--- a/test/metabase/xrays/automagic_dashboards/filters_test.clj
+++ b/test/metabase/xrays/automagic_dashboards/filters_test.clj
@@ -1,35 +1,37 @@
(ns metabase.xrays.automagic-dashboards.filters-test
(:require
[clojure.test :refer :all]
+ [metabase.lib.core :as lib]
[metabase.xrays.automagic-dashboards.filters :as filters]))
(deftest ^:parallel replace-date-range-test
(testing "Replace range with the more specific `:=`."
- (is (= [:and
- [:= [:field 2 nil] 42]
- [:= [:field 9 {:source-field 1}] "foo"]]
- (filters/inject-refinement
- [:and
- [:= [:field 9 {:source-field 1}] "foo"]
- [:and
- [:> [:field 2 nil] 10]
- [:< [:field 2 nil] 100]]]
- [:= [:field 2 nil] 42])))))
+ (is (=? [[:= {} [:field {:source-field 1} 9] "foo"]
+ [:= {} [:field {} 2] 42]]
+ (filters/inject-refinement
+ (mapv
+ lib/normalize
+ [[:= [:field 9 {:source-field 1}] "foo"]
+ [:and
+ [:> [:field 2 nil] 10]
+ [:< [:field 2 nil] 100]]])
+ (lib/normalize
+ [:= [:field 2 nil] 42]))))))
(deftest ^:parallel merge-using-and-test
(testing "If there's no overlap between filter clauses, just merge using `:and`."
- (is (= [:and
- [:= [:field 3 nil] 42]
- [:= [:field 9 {:source-field 1}] "foo"]
- [:> [:field 2 nil] 10]
- [:< [:field 2 nil] 100]]
- (filters/inject-refinement
- [:and
- [:= [:field 9 {:source-field 1}] "foo"]
- [:and
- [:> [:field 2 nil] 10]
- [:< [:field 2 nil] 100]]]
- [:= [:field 3 nil] 42])))))
+ (is (=? [[:= {} [:field {:source-field 1} 9] "foo"]
+ [:> {} [:field {} 2] 10]
+ [:< {} [:field {} 2] 100]
+ [:= {} [:field {} 3] 42]]
+ (filters/inject-refinement
+ (mapv
+ lib/normalize
+ [[:= [:field 9 {:source-field 1}] "foo"]
+ [:and
+ [:> [:field 2 nil] 10]
+ [:< [:field 2 nil] 100]]])
+ (lib/normalize [:= [:field 3 nil] 42]))))))
(deftest ^:parallel interesting-fields-test
(testing "Should return only :type/Temporal and :type/Boolean fields, or fields with the :type/Category semantic type"
diff --git a/test/metabase/xrays/automagic_dashboards/interesting_test.clj b/test/metabase/xrays/automagic_dashboards/interesting_test.clj
index afcb0675e3a..b5e63830cd9 100644
--- a/test/metabase/xrays/automagic_dashboards/interesting_test.clj
+++ b/test/metabase/xrays/automagic_dashboards/interesting_test.clj
@@ -1,7 +1,6 @@
(ns metabase.xrays.automagic-dashboards.interesting-test
(:require
[clojure.test :refer :all]
- [metabase.models.interface :as mi]
[metabase.query-processor.metadata :as qp.metadata]
[metabase.test :as mt]
[metabase.xrays.automagic-dashboards.core :as magic]
@@ -9,20 +8,6 @@
[metabase.xrays.automagic-dashboards.util :as magic.util]
[toucan2.core :as t2]))
-;;; ------------------- `->reference` -------------------
-
-(deftest ^:parallel ->reference-test
- (is (= [:field 1 nil]
- (->> (assoc (mi/instance :model/Field) :id 1)
- (#'interesting/->reference :mbql))))
-
- (is (= [:field 2 {:source-field 1}]
- (->> (assoc (mi/instance :model/Field) :id 1 :fk_target_field_id 2)
- (#'interesting/->reference :mbql))))
- (is (= 42
- (->> 42
- (#'interesting/->reference :mbql)))))
-
;;; -------------------- Bind dimensions, candidate bindings, field candidates, and related --------------------
(defn field [table column]
@@ -267,7 +252,7 @@
:matches [{:name "Number of Greebles"}]}}]
b-lt-a-bindings [{"Quantity" {:score 100
:matches [{:name "Number of Nurnies"}]}}
- {"Quantity" {:score 1,
+ {"Quantity" {:score 1
:matches [{:name "Number of Greebles"}]}}]
bind-dimensions-merge-fn #(apply merge-with (fn [a b]
(case (compare (:score a) (:score b))
@@ -307,57 +292,57 @@
quantity-dim
unmatched-dim]
bindings (#'interesting/find-dimensions context dimensions)]
- (is (= {"Quantity" {:field_type [:entity/GenericTable :type/Quantity],
- :score 100,
- :matches [{:base_type :type/Integer,
- :name "Number of Nurnies",
- :semantic_type :type/Quantity,
- :link nil,
- :field_type [:entity/GenericTable :type/Quantity],
+ (is (= {"Quantity" {:field_type [:entity/GenericTable :type/Quantity]
+ :score 100
+ :matches [{:base_type :type/Integer
+ :name "Number of Nurnies"
+ :semantic_type :type/Quantity
+ :link nil
+ :field_type [:entity/GenericTable :type/Quantity]
:score 100}
- {:base_type :type/Integer,
- :name "Number of Greebles",
- :semantic_type :type/Quantity,
- :link nil,
- :field_type [:entity/GenericTable :type/Quantity],
- :score 100}]},
- "GenericNumber" {:field_type [:entity/GenericTable :type/Number],
- :score 80,
- :matches [{:base_type :type/Float,
- :name "A double number field",
- :link nil,
- :field_type [:entity/GenericTable :type/Number],
+ {:base_type :type/Integer
+ :name "Number of Greebles"
+ :semantic_type :type/Quantity
+ :link nil
+ :field_type [:entity/GenericTable :type/Quantity]
+ :score 100}]}
+ "GenericNumber" {:field_type [:entity/GenericTable :type/Number]
+ :score 80
+ :matches [{:base_type :type/Float
+ :name "A double number field"
+ :link nil
+ :field_type [:entity/GenericTable :type/Number]
:score 80}]}}
bindings)))))
(deftest ^:parallel bind-dimensions-select-most-specific-test
(testing "When multiple dimensions are candidates the most specific dimension is selected."
- (is (= {"Quantity" {:field_type [:entity/GenericTable :type/Quantity],
- :score 100,
- :matches [{:semantic_type :type/Quantity,
- :name "QUANTITY",
- :effective_type :type/Integer,
- :display_name "Quantity",
- :base_type :type/Integer,
- :link nil,
- :field_type [:entity/GenericTable :type/Quantity],
+ (is (= {"Quantity" {:field_type [:entity/GenericTable :type/Quantity]
+ :score 100
+ :matches [{:semantic_type :type/Quantity
+ :name "QUANTITY"
+ :effective_type :type/Integer
+ :display_name "Quantity"
+ :base_type :type/Integer
+ :link nil
+ :field_type [:entity/GenericTable :type/Quantity]
:score 100}]}}
(let [context {:source {:entity_type :entity/GenericTable
- :fields [{:semantic_type :type/Discount,
+ :fields [{:semantic_type :type/Discount
:name "DISCOUNT"
:effective_type :type/Float
:base_type :type/Float}
- {:semantic_type :type/Quantity,
+ {:semantic_type :type/Quantity
:name "QUANTITY"
:effective_type :type/Integer
:base_type :type/Integer}]}
:tables [{:entity_type :entity/TransactionTable
- :fields [{:semantic_type :type/Discount,
+ :fields [{:semantic_type :type/Discount
:name "DISCOUNT"
- :effective_type :type/Float,
+ :effective_type :type/Float
:display_name "Discount"
:base_type :type/Float}
- {:semantic_type :type/Quantity,
+ {:semantic_type :type/Quantity
:name "QUANTITY"
:effective_type :type/Integer
:display_name "Quantity"
@@ -539,49 +524,80 @@
(deftest ^:parallel grounded-metrics-test
(mt/dataset test-data
- (let [test-metrics [{:metric ["count"], :score 100, :metric-name "Count"}
- {:metric ["distinct" ["dimension" "FK"]], :score 100, :metric-name "CountDistinctFKs"}
- {:metric ["/"
- ["dimension" "Discount"]
- ["dimension" "Income"]], :score 100, :metric-name "AvgDiscount"}
- {:metric ["sum" ["dimension" "GenericNumber"]], :score 100, :metric-name "Sum"}
- {:metric ["avg" ["dimension" "GenericNumber"]], :score 100, :metric-name "Avg"}]
- total-field {:id 1 :name "TOTAL"}
- discount-field {:id 2 :name "DISCOUNT"}
- income-field {:id 3 :name "INCOME"}]
+ (let [test-metrics [{:metric ["count"], :score 100, :metric-name "Count"}
+ {:metric ["distinct" ["dimension" "FK"]], :score 100, :metric-name "CountDistinctFKs"}
+ {:metric ["/"
+ ["dimension" "Discount"]
+ ["dimension" "Income"]]
+ :score 100
+ :metric-name "AvgDiscount"}
+ {:metric ["sum" ["dimension" "GenericNumber"]], :score 100, :metric-name "Sum"}
+ {:metric ["avg" ["dimension" "GenericNumber"]], :score 100, :metric-name "Avg"}]
+ total-field (t2/instance :model/Field {:id 1, :name "TOTAL", :base_type :type/Number})
+ discount-field (t2/instance :model/Field {:id 2, :name "DISCOUNT", :base_type :type/Number})
+ income-field (t2/instance :model/Field {:id 3, :name "INCOME", :base_type :type/Number})]
(testing "When no dimensions are provided, we produce grounded dimensionless metrics"
- (is (= [{:metric-name "Count"
- :metric-title "Count"
- :metric-score 100
- :metric-definition {:aggregation [["count"]]}
+ (is (= [{:metric-name "Count"
+ :metric-title "Count"
+ :metric-score 100
+ :metric-definition {:xrays/aggregations [{:lib/type :lib/external-op, :operator :count, :args []}]}
:dimension-name->field {}}]
(interesting/grounded-metrics
test-metrics
{"Count" {:matches []}}))))
(testing "When we can match on a dimension, we produce every matching metric (2 for GenericNumber)"
- (is (=? [{:metric-name "Sum",
- :metric-definition {:aggregation [["sum" "TOTAL"]]}}
- {:metric-name "Avg",
- :metric-definition {:aggregation [["avg" "TOTAL"]]}}]
+ (is (=? [{:metric-name "Sum"
+ :metric-definition {:xrays/aggregations [{:lib/type :lib/external-op
+ :operator :sum
+ :args [{:lib/type :metadata/column
+ :id 1
+ :name "TOTAL"
+ :base-type :type/Number}]}]}}
+ {:metric-name "Avg"
+ :metric-definition {:xrays/aggregations [{:lib/type :lib/external-op
+ :operator :avg
+ :args [{:lib/type :metadata/column
+ :id 1
+ :name "TOTAL"
+ :base-type :type/Number}]}]}}]
(interesting/grounded-metrics
- ;; Drop Count
+ ;; Drop Count
(rest test-metrics)
{"Count" {:matches []}
"GenericNumber" {:matches [total-field]}}))))
(testing "The addition of Discount doesn't add more matches as we need
Income as well to add the metric that uses Discount"
(is (=? [{:metric-name "Sum"
- :metric-definition {:aggregation [["sum" "TOTAL"]]}}
+ :metric-definition {:xrays/aggregations [{:lib/type :lib/external-op
+ :operator :sum
+ :args [{:lib/type :metadata/column
+ :id 1
+ :name "TOTAL"
+ :base-type :type/Number}]}]}}
{:metric-name "Avg"
- :metric-definition {:aggregation [["avg" "TOTAL"]]}}]
+ :metric-definition {:xrays/aggregations [{:lib/type :lib/external-op
+ :operator :avg
+ :args [{:lib/type :metadata/column
+ :id 1
+ :name "TOTAL"
+ :base-type :type/Number}]}]}}]
(interesting/grounded-metrics
(rest test-metrics)
{"Count" {:matches []}
"GenericNumber" {:matches [total-field]}
"Discount" {:matches [discount-field]}}))))
(testing "Discount and Income will add the satisfied AvgDiscount grounded metric"
- (is (=? [{:metric-name "AvgDiscount",
- :metric-definition {:aggregation [["/" "DISCOUNT" "INCOME"]]}}
+ (is (=? [{:metric-name "AvgDiscount"
+ :metric-definition {:xrays/aggregations [{:lib/type :lib/external-op
+ :operator :/
+ :args [{:lib/type :metadata/column
+ :id 2
+ :name "DISCOUNT"
+ :base-type :type/Number}
+ {:lib/type :metadata/column
+ :id 3
+ :name "INCOME"
+ :base-type :type/Number}]}]}}
{:metric-name "Sum"}
{:metric-name "Avg"}]
(interesting/grounded-metrics
diff --git a/test/metabase/xrays/automagic_dashboards/names_test.clj b/test/metabase/xrays/automagic_dashboards/names_test.clj
index ab44c6685a2..b6aa36b0bdd 100644
--- a/test/metabase/xrays/automagic_dashboards/names_test.clj
+++ b/test/metabase/xrays/automagic_dashboards/names_test.clj
@@ -3,12 +3,15 @@
[clojure.string :as str]
[clojure.test :refer :all]
[java-time.api :as t]
+ [metabase.lib.core :as lib]
[metabase.test :as mt]
[metabase.util.date-2 :as u.date]
[metabase.util.i18n :refer [tru]]
+ [metabase.util.malli :as mu]
[metabase.xrays.api.automagic-dashboards :as api.automagic-dashboards]
[metabase.xrays.automagic-dashboards.core :as magic]
- [metabase.xrays.automagic-dashboards.names :as names]))
+ [metabase.xrays.automagic-dashboards.names :as names]
+ [metabase.xrays.automagic-dashboards.schema :as ads]))
;;; ------------------- Datetime humanization (for chart and dashboard titles) -------------------
@@ -36,7 +39,9 @@
:week-of-year (u.date/extract dt :week-of-year)}]
(testing (format "unit = %s" unit)
(is (= (str expected)
- (str (#'names/humanize-datetime t-str unit))))))))
+ (str (#'names/humanize-datetime t-str unit)))))))))
+
+(deftest ^:parallel temporal-humanization-test-2
(testing "Extracted unit handling"
(is (= :sunday (#'u.date/start-of-week)) "adjust the test, it assumes Sunday as first day of the week")
(let [t 3] ; t = 2 or t = 4 would also work
@@ -68,6 +73,10 @@
(is (some? (#'names/humanize-datetime "1990-09-09T12:30:00" unit)))))))
;;; ------------------- Cell titles -------------------
+
+(defn- cell-title [root cell-query]
+ (names/cell-title root (lib/normalize ::ads/root.cell-query cell-query)))
+
(deftest ^:parallel cell-title-test
(mt/$ids venues
(let [query (api.automagic-dashboards/adhoc-query-instance {:query {:source-table (mt/id :venues)
@@ -78,30 +87,30 @@
(testing "Should humanize equal filter"
(is (= "number of Venues where Name is Test"
;; Test specifically the un-normalized form (metabase#15737)
- (names/cell-title root ["=" ["field" %name nil] "Test"]))))
+ (cell-title root ["=" ["field" %name nil] "Test"]))))
(testing "Should humanize greater than filter"
(is (= "number of Venues where Name is greater than Test"
- (names/cell-title root [">" ["field" %name nil] "Test"]))))
+ (cell-title root [">" ["field" %name nil] "Test"]))))
(testing "Should humanize at least filter"
(is (= "number of Venues where Name is at least Test"
- (names/cell-title root [">=" ["field" %name nil] "Test"]))))
+ (cell-title root [">=" ["field" %name nil] "Test"]))))
(testing "Should humanize less than filter"
(is (= "number of Venues where Name is less than Test"
- (names/cell-title root ["<" ["field" %name nil] "Test"]))))
+ (cell-title root ["<" ["field" %name nil] "Test"]))))
(testing "Should humanize no more than filter"
(is (= "number of Venues where Name is no more than Test"
- (names/cell-title root ["<=" ["field" %name nil] "Test"]))))
+ (cell-title root ["<=" ["field" %name nil] "Test"]))))
(testing "Should humanize and filter"
(is (= "number of Venues where Name is Test and Price is 0"
- (names/cell-title root ["and"
- ["=" $name "Test"]
- ["=" $price 0]]))))
+ (cell-title root ["and"
+ ["=" $name "Test"]
+ ["=" $price 0]]))))
(testing "Should humanize between filter"
(is (= "number of Venues where Name is between A and J"
- (names/cell-title root ["between" $name "A", "J"]))))
+ (cell-title root ["between" $name "A", "J"]))))
(testing "Should humanize inside filter"
(is (= "number of Venues where Longitude is between 2 and 4; and Latitude is between 3 and 1"
- (names/cell-title root ["inside" (mt/$ids venues $latitude) (mt/$ids venues $longitude) 1 2 3 4])))))))
+ (cell-title root ["inside" (mt/$ids venues $latitude) (mt/$ids venues $longitude) 1 2 3 4])))))))
(deftest ^:parallel cell-title-default-test
(mt/$ids venues
@@ -111,8 +120,9 @@
:database (mt/id)})
root (magic/->root query)]
(testing "Just say \"relates to\" when we don't know what the operator is"
- (is (= "number of Venues where Name relates to Test"
- (names/cell-title root ["???" ["field" %name nil] "Test"])))))))
+ (mu/disable-enforcement
+ (is (= "number of Venues where Name relates to Test"
+ (cell-title root ["???" [:field {:lib/uuid (str (random-uuid))} %name] "Test"]))))))))
(deftest ^:parallel cell-title-with-dates-comparison-test
(testing "Ensure cell titles with date comparisons display correctly"
@@ -126,24 +136,24 @@
;; humanize-datetime which we probably should not do right now.
(testing "Should humanize temporal field with = test"
(is (= "number of Users where Last Login is on September 9, 1990"
- (names/cell-title root
- ["=" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))
+ (cell-title root
+ ["=" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))
(testing "Should humanize temporal field with > test"
(is (= "number of Users where Last Login is after on September 9, 1990"
- (names/cell-title root
- [">" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))
+ (cell-title root
+ [">" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))
(testing "Should humanize temporal field with < test"
(is (= "number of Users where Last Login is before on September 9, 1990"
- (names/cell-title root
- ["<" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))
+ (cell-title root
+ ["<" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))
(testing "Should humanize temporal field with >= test"
(is (= "number of Users where Last Login is not before on September 9, 1990"
- (names/cell-title root
- [">=" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))
+ (cell-title root
+ [">=" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))
(testing "Should humanize temporal field with <= test"
(is (= "number of Users where Last Login is not after on September 9, 1990"
- (names/cell-title root
- ["<=" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))))))
+ (cell-title root
+ ["<=" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))))))
(deftest ^:parallel no-null-cell-title-temporal-units-35170-test
(testing "Ensure various permutations of temporal units produce valid strings (without nulls in them)
@@ -156,25 +166,25 @@
root (magic/->root query)]
(testing "Should not contain nulls when temporal-unit is hour"
(is (= "number of Users where Last Login is at 12 PM, September 9, 1990"
- (names/cell-title root
- ["=" ["field" %last_login {:temporal-unit :hour}] "1990-09-09T12:30:00"]))))
+ (cell-title root
+ ["=" ["field" %last_login {:temporal-unit :hour}] "1990-09-09T12:30:00"]))))
(testing "Should not contain nulls when temporal-unit is day"
(is (= "number of Users where Last Login is on January 1, 2020"
- (names/cell-title root
- ["=" ["field" %last_login {:temporal-unit :day}] "2020"]))))
+ (cell-title root
+ ["=" ["field" %last_login {:temporal-unit :day}] "2020"]))))
(testing "Should not contain nulls when temporal-unit is month"
(is (= "number of Users where Last Login is in January 2020"
- (names/cell-title root
- ["=" ["field" %last_login {:temporal-unit :month}] "2020"]))))
+ (cell-title root
+ ["=" ["field" %last_login {:temporal-unit :month}] "2020"]))))
(testing "Should not contain nulls when temporal-unit is quarter"
(is (= "number of Users where Last Login is in Q1 - 2020"
- (names/cell-title root
- ["=" ["field" %last_login {:temporal-unit :quarter}] "2020"]))))
+ (cell-title root
+ ["=" ["field" %last_login {:temporal-unit :quarter}] "2020"]))))
;; This was producing "number of Users where null of Last Login is 2020" originally
(testing "Should not contain nulls when temporal-unit is year"
(is (= "number of Users where year of Last Login is 2020"
- (names/cell-title root
- ["=" ["field" %last_login {:temporal-unit :year}] "2020"]))))))))
+ (cell-title root
+ ["=" ["field" %last_login {:temporal-unit :year}] "2020"]))))))))
(deftest ^:parallel humanize-filter-value-works-for-expressions-16680-test
(testing "Expressions can be used in humanized names in addition to fields"
@@ -182,8 +192,12 @@
(is (= "TestColumn is 2 and Created At is in February 2024"
(#'names/humanize-filter-value
{:database (mt/id)}
- ["and"
- ["=" ["expression" "TestColumn" {:base-type "type/Integer"}] 2]
- ["=" ["field" (mt/id :orders :created_at)
- {:base-type "type/DateTime", :temporal-unit "month"}]
+ [:and
+ {:lib/uuid "433ee131-2210-4b94-88fb-ff4967eaa555"}
+ [:= {:lib/uuid "02a0746d-cf2d-4878-b5e6-26bce3e81986"}
+ [:expression {:base-type :type/Integer, :lib/uuid "9ec2cd31-02b9-404d-a63b-681a140b9498"} "TestColumn"]
+ 2]
+ [:= {:lib/uuid "88ddba2f-5047-4b31-9c7f-6c0b5bc8ffd9"}
+ [:field {:base-type :type/DateTime, :temporal-unit :month, :lib/uuid "c40000a9-da61-410d-8822-0b888fd2396a"}
+ (mt/id :orders :created_at)]
"2024-02-01T00:00:00Z"]]))))))
diff --git a/test/metabase/xrays/automagic_dashboards/schema_test.clj b/test/metabase/xrays/automagic_dashboards/schema_test.clj
new file mode 100644
index 00000000000..bb48789d3ba
--- /dev/null
+++ b/test/metabase/xrays/automagic_dashboards/schema_test.clj
@@ -0,0 +1,26 @@
+(ns metabase.xrays.automagic-dashboards.schema-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.lib.core :as lib]
+ [metabase.xrays.automagic-dashboards.schema :as ads]))
+
+(deftest ^:parallel normalize-cell-query-test
+ (is (=? [:=
+ {:lib/uuid string?}
+ [:field {:base-type :type/Text, :lib/uuid string?} "SOURCE"]
+ "Affiliate"]
+ (lib/normalize ::ads/root.cell-query ["=" ["field" "SOURCE" {"base-type" "type/Text"}] "Affiliate"])))
+ (is (=? [:and
+ {:lib/uuid string?}
+ [:= {:lib/uuid string?}
+ [:expression {:lib/uuid string?} "TestColumn"]
+ 2]
+ [:= {:lib/uuid string?}
+ [:field {:temporal-unit :month, :lib/uuid string?}
+ 13]
+ "2019-02-01T00:00:00Z"]]
+ (lib/normalize ::ads/root.cell-query [:and
+ [:= [:expression "TestColumn"] 2]
+ [:=
+ [:field 13 {:temporal-unit :month}]
+ "2019-02-01T00:00:00Z"]]))))
diff --git a/test/metabase/xrays/automagic_dashboards/util_test.clj b/test/metabase/xrays/automagic_dashboards/util_test.clj
index 2c0683a2aff..cddcc9b4d45 100644
--- a/test/metabase/xrays/automagic_dashboards/util_test.clj
+++ b/test/metabase/xrays/automagic_dashboards/util_test.clj
@@ -19,7 +19,9 @@
(is (=? {:id (mt/id :orders :discount)}
(magic.util/->field root (mt/id :orders :discount))))
(is (=? {:id (mt/id :orders :discount)}
- (magic.util/->field root [:field (mt/id :orders :discount) nil]))))))))))
+ (magic.util/->field
+ root
+ [:field {:lib/uuid (str (random-uuid))} (mt/id :orders :discount)]))))))))))
(deftest ^:parallel ->field-test-2
(testing "Demonstrate the stated methods in which ->fields works"
@@ -33,7 +35,9 @@
(is (=? {:id (mt/id :orders :discount)}
(magic.util/->field root (mt/id :orders :discount))))
(is (=? {:id (mt/id :orders :discount)}
- (magic.util/->field root [:field (mt/id :orders :discount) nil]))))
+ (magic.util/->field
+ root
+ [:field {:lib/uuid (str (random-uuid))} (mt/id :orders :discount)]))))
(testing "Looking up the field by name or named field ref works,
returning the metadata description of the field."
(is (=? {:name "DISCOUNT"
@@ -41,4 +45,6 @@
(magic.util/->field root "DISCOUNT"))))
(is (=? {:name "DISCOUNT"
:field_ref [:field "DISCOUNT" {:base-type :type/Float}]}
- (magic.util/->field root [:field "DISCOUNT" {:base-type :type/Float}])))))))))))
+ (magic.util/->field
+ root
+ [:field {:base-type :type/Float, :lib/uuid (str (random-uuid))} "DISCOUNT"])))))))))))