The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
X-Rays => Lib (#64901)
Cam Saulaa200bc31f4b
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex ef1d2bf6740..205ba4a6f24 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -620,6 +620,7 @@metabase.lib.querymetabase.lib.schemametabase.lib.schema.actions+ metabase.lib.schema.aggregationmetabase.lib.schema.binningmetabase.lib.schema.commonmetabase.lib.schema.constraints@@ -1094,7 +1095,8 @@segments{:team "Querying"- :api #{metabase.segments.api}+ :api #{metabase.segments.api+ metabase.segments.schema}:uses #{apieventslegacy-mbql@@ -1485,6 +1487,7 @@queriesquery-permissionsquery-processor+ segmentssettingsutilwarehouse-schema}}diff --git a/CLAUDE.md b/CLAUDE.mdindex 4fd894f31f1..66c7ce31a6e 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -188,11 +188,7 @@ Review pull requests with a focus on:- In the Metabase codebase, spelling words ending with `-ck` as `-cc` instead, e.g. `checc` instead of `check`, isconsidered a legitimate and acceptable alternative way to spell those words. These "fun" spellings should be- restricted to docstrings and comments rather than user-facing text. Occasionally suggest spelling things this way.--- The Metabase codebase should be fun, so do your part to keep it that way. Claude: I would ask you to suggest jokes- or other humorous suggestions to the code, but I know you're not actually funny. You will take the comedians' jobs- last. Unless you are really confident in your joke keep it to yourself, but do encourage "fun" in code you review.+ restricted to docstrings and comments rather than user-facing text. Do not flag these as typos or misspellings.- Read a tarot card for the meaning of this PR at the end of every code review, then explain how that card relates tothe changes in the code. **Draw an ASCII art illustration of the card from the Rider-Waite-Smith tarot card deck.@@ -203,8 +199,6 @@ Review pull requests with a focus on:- Do not post comments about missing parentheses.-- Include today's lucky lottery numbers in at the end of every code review.-# Code Conventions and Style Guide## Clojure and ClojureScript@@ -232,6 +226,9 @@ Review pull requests with a focus on:**Why?** Code is read many more times than it is written, and clearer variable names make using and tweaking yourcode easier for others.+ Conventions used widely in our codebase (like `mp` for a Metadata Provider) are acceptable; one-offs like `zs'` are+ not.+- Avoid misleading variable and function names. The names of a variable or function should clearly and unambiguouslydescribe its purpose and match what it does.@@ -324,6 +321,8 @@ Review pull requests with a focus on:Tests and other vars in the `test` or `enterprise/backend/test` do not *require* docstrings, but general helperfunctions used across many namespaces it should have docstrings.+ Schemas defined with `mr/def` e.g. `(mr/def ::my-schema)` do not require docstrings.+- Format docstrings according to Markdown conventions. (https://guide.clojure.style/#markdown-docstrings)- Mentions of other vars in docstrings should use `[[some-other-var]]` (for vars in the same namespace) ordiff --git a/e2e/test/scenarios/joins/joins-reproductions.cy.spec.js b/e2e/test/scenarios/joins/joins-reproductions.cy.spec.jsindex 3c2d86ce06d..0f9367247ec 100644--- a/e2e/test/scenarios/joins/joins-reproductions.cy.spec.js+++ b/e2e/test/scenarios/joins/joins-reproductions.cy.spec.js@@ -165,7 +165,7 @@ describe("issue 14793", () => {.findByText(/^A closer look at/).should("be.visible");- H.getDashboardCards().should("have.length", 18);+ H.getDashboardCards().should("have.length", 35);});});diff --git a/src/metabase/lib/aggregation.cljc b/src/metabase/lib/aggregation.cljcindex ce02411a2b7..47d36a45113 100644--- a/src/metabase/lib/aggregation.cljc+++ b/src/metabase/lib/aggregation.cljc@@ -10,6 +10,7 @@[metabase.lib.metadata.calculation :as lib.metadata.calculation][metabase.lib.options :as lib.options][metabase.lib.ref :as lib.ref]+ [metabase.lib.remove-replace :as lib.remove-replace][metabase.lib.schema :as lib.schema][metabase.lib.schema.aggregation :as lib.schema.aggregation][metabase.lib.schema.common :as lib.schema.common]@@ -21,6 +22,7 @@[metabase.util :as u][metabase.util.i18n :as i18n][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.performance :refer [select-keys mapv #?(:clj doseq) #?(:clj for)]]))(mu/defn column-metadata->aggregation-ref :- :mbql.clause/aggregation@@ -283,12 +285,12 @@[aggregation-clause]aggregation-clause)-(def ^:private Aggregable+(mr/def ::aggregable"Schema for something you can pass to [[aggregate]] to add to a query as an aggregation."[:or- ::lib.schema.aggregation/aggregation- ::lib.schema.common/external-op- ::lib.schema.metadata/metric])+ [:ref ::lib.schema.aggregation/aggregation]+ [:ref ::lib.schema.common/external-op]+ [:ref ::lib.schema.metadata/metric]])(mu/defn aggregate :- ::lib.schema/query"Adds an aggregation to query."@@ -297,7 +299,7 @@([query :- ::lib.schema/querystage-number :- :int- aggregable :- Aggregable]+ aggregable :- ::aggregable];; if this is a Metric metadata, convert it to `:metric` MBQL clause before adding.(if (= (lib.dispatch/dispatch-value aggregable) :metadata/metric)(recur query stage-number (lib.ref/ref aggregable))@@ -481,3 +483,16 @@[query stage-number ag-ref](let [expression (resolve-aggregation query stage-number ag-ref)](lib.metadata.calculation/type-of query stage-number expression)))++(mu/defn remove-all-aggregations :- ::lib.schema/query+ "Remove all aggregations from a query stage."+ ([query]+ (remove-all-aggregations query -1))++ ([query :- ::lib.schema/query+ stage-number :- :int]+ (reduce+ (fn [query ag]+ (lib.remove-replace/remove-clause query stage-number ag))+ query+ (aggregations query stage-number))))diff --git a/src/metabase/lib/common.cljc b/src/metabase/lib/common.cljcindex c20b52a0676..300a1de6354 100644--- a/src/metabase/lib/common.cljc+++ b/src/metabase/lib/common.cljc@@ -61,9 +61,10 @@(defmethod ->op-arg :lib/external-op[{:keys [operator options args] :or {options {}}}]- (->op-arg (lib.options/ensure-uuid (into [(keyword operator) options]- (map ->op-arg)- args))))+ (->op-arg (-> (lib.options/ensure-uuid (into [(keyword operator) options]+ (map ->op-arg)+ args))+ ((#?(:clj requiring-resolve :cljs resolve) 'metabase.lib.normalize/normalize)))))(defn defop-create"Impl for [[defop]]."diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex 83ff2ab4e58..db96841c42b 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -127,6 +127,7 @@aggregationsaggregations-metadataavailable-aggregation-operators+ remove-all-aggregationsselected-aggregation-operatorscountavgdiff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljcindex 5d8a7b0bd64..3563ba4306b 100644--- a/src/metabase/lib/schema.cljc+++ b/src/metabase/lib/schema.cljc@@ -143,18 +143,26 @@acc))stage stage))+(def ^:dynamic *HACK-disable-ref-validation*+ "Whether to validate join aliases in field refs and expression refs. This is only disable-able as a hack to support+ X-Rays code which generates fragments of stages that drop joins and expressions and then adds them again after the+ fact in [[metabase.xrays.automagic-dashboards.core/preserve-entity-element]]. Once we port X-Rays to use Lib we can+ fix the hackiness and hopefully take this out."+ false)+(defn- expression-ref-errors-for-stage [stage]- (let [stage (dissoc stage :parameters) ; don't validate [:dimension [:expression ...]] refs since they might not be moved to the correct place yet.- expression-names (when-let [expressions (:expressions stage)]- (when (and (sequential? expressions)- (every? sequential? expressions))- (into #{} (map (comp :lib/expression-name second)) expressions)))- pred #(bad-ref-clause? :expression expression-names %)- form (-> (stage-with-joins-and-namespaced-keys-removed stage)- ;; also ignore expression refs inside `:parameters` since they still use legacy syntax these days.- (dissoc :parameters))]- (when (lib.schema.util/pred-matches-form? form pred)- (lib.schema.util/matching-locations form pred))))+ (when-not *HACK-disable-ref-validation*+ (let [stage (dissoc stage :parameters) ; don't validate [:dimension [:expression ...]] refs since they might not be moved to the correct place yet.+ expression-names (when-let [expressions (:expressions stage)]+ (when (and (sequential? expressions)+ (every? sequential? expressions))+ (into #{} (map (comp :lib/expression-name second)) expressions)))+ pred #(bad-ref-clause? :expression expression-names %)+ form (-> (stage-with-joins-and-namespaced-keys-removed stage)+ ;; also ignore expression refs inside `:parameters` since they still use legacy syntax these days.+ (dissoc :parameters))]+ (when (lib.schema.util/pred-matches-form? form pred)+ (lib.schema.util/matching-locations form pred)))))(defn- aggregation-ref-errors-for-stage [stage](let [uuids (into #{} (map (comp :lib/uuid second)) (:aggregation stage))@@ -348,17 +356,10 @@(mapcat join-aliases-in-join (:joins stage)))](set (join-aliases-in-stage stage)))))-(def ^:dynamic *HACK-disable-join-alias-in-field-ref-validation*- "Whether to validate join aliases in field refs. This is only disable-able as a hack to support X-Rays code which- generates fragments of stages that drop joins and then adds them again after the fact- in [[metabase.xrays.automagic-dashboards.core/preserve-entity-element]]. Once we port X-Rays to use Lib we can fix- the hackiness and hopefully take this out."- false)-(defn- join-ref-error-for-stages"Return an error messages if we find a field ref that uses a `:join-alias` for a join that doesn't exist."[stages]- (when (and (not *HACK-disable-join-alias-in-field-ref-validation*)+ (when (and (not *HACK-disable-ref-validation*)(sequential? stages))(loop [visible-join-alias? (constantly false), i 0, [stage & more] stages](let [visible-join-alias? (some-fn visible-join-alias? (visible-join-alias?-fn stage))]diff --git a/src/metabase/lib/schema/common.cljc b/src/metabase/lib/schema/common.cljcindex aae19f9d3c6..df27ee626af 100644--- a/src/metabase/lib/schema/common.cljc+++ b/src/metabase/lib/schema/common.cljc@@ -287,8 +287,8 @@[:operator [:multi {:dispatch string?}[true :string][false :keyword]]]- [:args [:sequential :any]]- [:options {:optional true} ::options]])+ [:args [:schema {:decode/normalize vec} [:sequential :any]]]+ [:options {:optional true} ::options]])#?(:clj(defn- instance-of-class* [& classes]diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.cljindex 26ecf0756a1..8bc93fe3fde 100644--- a/src/metabase/lib_be/metadata/jvm.clj+++ b/src/metabase/lib_be/metadata/jvm.clj@@ -45,10 +45,11 @@(def ^:private metadata-type->schema{:metadata/card ::lib.schema.metadata/card})-(defn instance->metadata+(mu/defn instance->metadata"Convert a (presumably) Toucan 2 instance of an application database model with `snake_case` keys to a MLv2 stylemetadata instance with `:lib/type` and `kebab-case` keys."- [instance metadata-type]+ [instance :- :map+ metadata-type :- :keyword](let [normalize (if-let [schema (get metadata-type->schema metadata-type)](fn [instance](lib.normalize/normalize schema instance))diff --git a/src/metabase/query_processor/store.clj b/src/metabase/query_processor/store.cljindex b058becfe68..b934f8dd3a5 100644--- a/src/metabase/query_processor/store.clj+++ b/src/metabase/query_processor/store.clj@@ -216,7 +216,7 @@;;;; DEPRECATED STUFF;;;;-(defn ->legacy-metadata+(mu/defn ->legacy-metadata"For compatibility: convert MLv2-style metadata as returned by [[metabase.lib.metadata.protocols]]or [[metabase.lib.metadata]] functions(with `kebab-case` keys and `:lib/type`) to legacy QP/application database style metadata (with `snake_case` keys@@ -227,11 +227,8 @@(Note: it is preferable to use [[metabase.lib.core/lib-metadata-column->legacy-metadata-column]] instead of thisfunction if you REALLY need to do this sort of conversion.)"{:deprecated "0.48.0"}- [lib-metadata-col]- (let [model (case (:lib/type lib-metadata-col)- :metadata/database :model/Database- :metadata/table :model/Table- :metadata/column :model/Field)]- (-> lib-metadata-col- lib/lib-metadata-column->legacy-metadata-column- (vary-meta assoc :type model))))+ [lib-metadata-col :- [:map+ [:lib/type [:= :metadata/column]]]]+ (-> lib-metadata-col+ lib/lib-metadata-column->legacy-metadata-column+ (vary-meta assoc :type :metadata/column)))diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.cljindex b5a4753edfd..17e85e549c5 100644--- a/src/metabase/segments/models/segment.clj+++ b/src/metabase/segments/models/segment.clj@@ -7,7 +7,6 @@[metabase.api.common :as api];; existing usages, do not use legacy MBQL utils in new code^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]^{:clj-kondo/ignore [:discouraged-namespace :deprecated-namespace]} [metabase.legacy-mbql.util :as mbql.u][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]@@ -20,6 +19,7 @@[metabase.models.serialization :as serdes][metabase.permissions.core :as perms][metabase.search.core :as search]+ [metabase.segments.schema :as segments.schema][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.log :as log]@@ -33,14 +33,9 @@(methodical/defmethod t2/table-name :model/Segment [_model] :segment)(methodical/defmethod t2/model-for-automagic-hydration [:default :segment] [_original-model _k] :model/Segment)-(mr/def ::segment-definition- [:map- [:filter {:optional true} [:maybe mbql.s/Filter]]- [:aggregation {:optional true} [:maybe [:sequential ::mbql.s/Aggregation]]]])-(defn- validate-segment-definition[definition]- (if-let [error (mr/explain ::segment-definition definition)]+ (if-let [error (mr/explain ::segments.schema/segment definition)](let [humanized (me/humanize error)](throw (ex-info (tru "Invalid Metric or Segment: {0}" (pr-str humanized)){:error errordiff --git a/src/metabase/segments/schema.clj b/src/metabase/segments/schema.cljnew file mode 100644index 00000000000..faaeda03f10--- /dev/null+++ b/src/metabase/segments/schema.clj@@ -0,0 +1,10 @@+(ns metabase.segments.schema+ (:require+ ;; existing usage, do not use legacy MBQL utils in new code+ ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]+ [metabase.util.malli.registry :as mr]))++(mr/def ::segment+ [:map+ [:filter {:optional true} [:maybe mbql.s/Filter]]+ [:aggregation {:optional true} [:maybe [:sequential ::mbql.s/Aggregation]]]])diff --git a/src/metabase/xrays/api/automagic_dashboards.clj b/src/metabase/xrays/api/automagic_dashboards.cljindex 2d2cb3e5259..b2102b19484 100644--- a/src/metabase/xrays/api/automagic_dashboards.clj+++ b/src/metabase/xrays/api/automagic_dashboards.clj@@ -5,6 +5,8 @@[metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.models.interface :as mi][metabase.queries.core :as queries][metabase.query-permissions.core :as query-perms]@@ -263,11 +265,12 @@:keys [linked-tables]}](if (seq linked-tables)(let [child-dashboards (map (fn [{:keys [linked-table-id linked-field-id]}]- (let [table (t2/select-one :model/Table :id linked-table-id)]+ (let [table (t2/select-one :model/Table :id linked-table-id)+ mp (lib-be/application-database-metadata-provider (:db_id table))](automagic-dashboards.core/automagic-analysistable{:show :all- :query-filter [:= [:field linked-field-id nil] model_pk]})))+ :query-filter [(lib/= (lib.metadata/field mp linked-field-id) model_pk)]})))linked-tables)seed-dashboard (-> (first child-dashboards)(mergediff --git a/src/metabase/xrays/automagic_dashboards/combination.clj b/src/metabase/xrays/automagic_dashboards/combination.cljindex 7a92eba6d3d..d8673ae110f 100644--- a/src/metabase/xrays/automagic_dashboards/combination.clj+++ b/src/metabase/xrays/automagic_dashboards/combination.clj@@ -21,12 +21,12 @@[clojure.walk :as walk][medley.core :as m][metabase.driver.util :as driver.u]- ;; legacy usage, do not use this in new code- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]- [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.models.interface :as mi][metabase.queries.core :as queries]- [metabase.query-processor.util :as qp.util][metabase.util :as u][metabase.util.i18n :as i18n][metabase.util.malli :as mu]@@ -36,19 +36,20 @@[metabase.xrays.automagic-dashboards.util :as magic.util][metabase.xrays.automagic-dashboards.visualization-macros :as visualization]))-(defn add-breakouts-and-filter+(mu/defn add-breakouts-and-filters :- ::ads/grounded-metric.definition"Add breakouts and filters to a query based on the breakout fields and filter clauses"- [query- breakout-fields- filter-clauses]- (let [breakouts (mapv (partial interesting/->reference :mbql) breakout-fields)]- (cond-> (assoc query :breakout breakouts)- (seq filter-clauses) (assoc :filter (into [:and] filter-clauses)))))--(defn- add-aggregations+ [metric-def :- ::ads/grounded-metric.definition+ breakouts :- [:maybe [:sequential ::lib.schema.metadata/column]]+ filter-clauses :- [:maybe [:sequential ::ads/external-op]]]+ (assoc metric-def+ :xrays/breakouts breakouts+ :xrays/filters filter-clauses))++(mu/defn- add-aggregations :- ::ads/grounded-metric.definition"Add aggregations to a query."- [query aggregations]- (assoc query :aggregation aggregations))+ [metric-def :- ::ads/grounded-metric.definition+ aggregations :- [:sequential ::ads/aggregation]]+ (assoc metric-def :xrays/aggregations aggregations))(defn matching-types?"Given two seqs of types, return true of the types of the child@@ -82,21 +83,29 @@[:dataset_query ::ads/query]]"Add the `:dataset_query` key to this metric. Requires both the current metric-definition (from the grounded metric)and the database and table ids (from the source object)."- [{:keys [metric-definition] :as ground-metric-with-dimensions}+ [{:keys [metric-definition] :as ground-metric-with-dimensions} :- ::ads/grounded-metric{{:keys [database]} :root :keys [source query-filter]} :- ::ads/context]- (let [source-table (if (->> source (mi/instance-of? :model/Table))- (-> source u/the-id)- (->> source u/the-id (str "card__")))- model? (and (mi/instance-of? :model/Card source)- (queries/model? source))]- (assoc ground-metric-with-dimensions- :dataset_query (-> {:database database- :type :query- :query (cond-> (assoc metric-definition- :source-table source-table)- (and (not model?)- query-filter) (assoc :filter query-filter))}- mbql.normalize/normalize))))+ (let [model? (and (mi/instance-of? :model/Card source)+ (queries/model? source))+ mp (lib-be/application-database-metadata-provider database)+ query (cond-> (lib/query mp (cond+ (mi/instance-of? :model/Table source) (lib.metadata/table mp (u/the-id source))+ (mi/instance-of? :model/Card source) (lib.metadata/card mp (u/the-id source))))+ (:xrays/aggregations metric-definition)+ (as-> $query (reduce lib/aggregate $query (:xrays/aggregations metric-definition)))++ (:xrays/breakouts metric-definition)+ (as-> $query (reduce lib/breakout $query (:xrays/breakouts metric-definition)))++ (:xrays/filters metric-definition)+ (as-> $query (reduce lib/filter $query (:xrays/filters metric-definition)))++ (not model?)+ (as-> $query (reduce+ lib/filter+ $query+ query-filter)))]+ (assoc ground-metric-with-dimensions :dataset_query query)))(defn- instantiate-visualization[[k v] dimensions metrics]@@ -125,7 +134,7 @@(str/replace s #"\[\[(\w+)(?:\.([\w\-]+))?\]\]"(fn [[_ identifier attribute]](let [entity (binding-fn identifier)- attribute (some-> attribute qp.util/normalize-token)]+ attribute (some-> attribute u/->kebab-case-en keyword)](str (or (and (ifn? entity) (entity attribute))(root attribute)(interesting/->reference template-type entity))))))))@@ -156,22 +165,6 @@dimension-name->field(map first card-dimensions)))-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(def ^:private ^{:arglists '([field])} id-or-name- (some-fn :id :name))--(defn- singular-cell-dimensions- [{:keys [cell-query]}]- (letfn [(collect-dimensions [[op & args]]- (case (some-> op qp.util/normalize-token)- :and (mapcat collect-dimensions args)- (:between :=) (magic.util/collect-field-references args)- nil))]- (->> cell-query- collect-dimensions- (map magic.util/field-reference->id)- set)))-(defn- valid-breakout-dimension?[{:keys [base_type db fingerprint aggregation]}](or (nil? aggregation)@@ -181,30 +174,25 @@(not= (-> fingerprint :type :type/Number :min)(-> fingerprint :type :type/Number :max)))))-(defn- valid-bindings? [{:keys [root]} satisfied-dimensions bindings]- (let [cell-dimension? (singular-cell-dimensions root)]- (->> satisfied-dimensions- (map first)- (map (fn [[identifier opts]]- (merge (bindings identifier) opts)))- (every? (every-pred valid-breakout-dimension?- (complement (comp cell-dimension? id-or-name)))))))-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;+(defn- valid-bindings? [satisfied-dimensions bindings]+ (->> satisfied-dimensions+ (map first)+ (map (fn [[identifier opts]]+ (merge (bindings identifier) opts)))+ (every? valid-breakout-dimension?)))(mu/defn grounded-metrics->dashcards :- [:sequential[:merge- ads/combined-metric+ ::ads/combined-metric[:map- [:dataset_query- [:map- [:database ::lib.schema.id/database]]]]]]+ [:dataset_query ::ads/query]]]]"Generate dashcards from ground dimensions, using the base context, ground dimensions,card templates, and grounded metrics as input."[base-context :- ::ads/contextcard-templates- ground-dimensions :- ads/dim-name->matching-fields- ground-filters- grounded-metrics :- [:sequential ads/grounded-metric]]+ ground-dimensions :- ::ads/dim-name->matching-fields+ ground-filters :- [:sequential ::ads/grounded-filter]+ grounded-metrics :- [:sequential ::ads/grounded-metric]](let [metric-name->metric (zipmap(map :metric-name grounded-metrics)(map-indexed@@ -226,12 +214,11 @@(apply math.combo/cartesian-product)(map (partial zipmap dim-names))):let [merged-dims (combine-dimensions dimension-name->field card-dimensions)]- :when (and (valid-bindings? base-context card-dimensions dimension-name->field)+ :when (and (valid-bindings? card-dimensions dimension-name->field)(every? metric-name->metric card-metrics)):let [[grounded-metric :as all-satisfied-metrics] (map metric-name->metric card-metrics)final-aggregate (into []- (comp (map (comp :aggregation :metric-definition))- cat)+ (mapcat (comp :xrays/aggregations :metric-definition))all-satisfied-metrics)bound-metric-dimension-name->field (apply merge (map :dimension-name->field all-satisfied-metrics))all-names->field (into dimension-name->field bound-metric-dimension-name->field)@@ -249,15 +236,18 @@card(-> grounded-metric(assoc- :id (gensym)+ :id (gensym):affinity-name card-name- :card-score card-score- :total-score (long (/ (apply + score-components) (count score-components)))+ :card-score card-score+ :total-score (long (/ (apply + score-components) (count score-components)));; Update dimension-name->field to include named contributions from both metrics and dimensions:dimension-name->field all-names->field- :score-components score-components)- (update :metric-definition add-aggregations final-aggregate)- (update :metric-definition add-breakouts-and-filter- (vals merged-dims)- (mapv (comp :filter simple-grounded-filters) card-filters))+ :score-components score-components)+ (update :metric-definition (fn [metric-definition]+ (-> metric-definition+ (add-aggregations final-aggregate)+ (add-breakouts-and-filters (for [field (vals merged-dims)]+ (interesting/field->metadata field))+ (mapv (comp :filter simple-grounded-filters)+ card-filters)))))(add-dataset-query base-context))))))diff --git a/src/metabase/xrays/automagic_dashboards/comparison.clj b/src/metabase/xrays/automagic_dashboards/comparison.cljindex 519db963352..08bd000a048 100644--- a/src/metabase/xrays/automagic_dashboards/comparison.clj+++ b/src/metabase/xrays/automagic_dashboards/comparison.clj@@ -2,8 +2,9 @@(:require[medley.core :as m][metabase.api.common :as api]- ;; legacy usage, do not use this in new code- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.lib.core :as lib]+ [metabase.lib.equality :as lib.equality]+ [metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.util :as u]@@ -53,36 +54,30 @@instead, but that validates its input and output, and the queries that come in here aren't always valid (forexample, missing `:database`). If we can, it would be nice to use that instead of reinventing the wheel here."[query :- ::ads/query- new-filter-clauses :- [:maybe [:sequential :any]]]- (magic.util/do-with-legacy-query- query- (fn [{{existing-filter-clause :filter} :query, :as query}]- (let [clauses (filter identity (cons existing-filter-clause new-filter-clauses))- new-filter-clause (when (seq clauses)- ;; existing usage, this namespace will be rewritten to use Lib soon.- #_{:clj-kondo/ignore [:deprecated-var]}- (mbql.normalize/normalize-fragment [:query :filter] (cons :and clauses)))]- (cond-> query- (seq new-filter-clause) (magic.util/do-with-legacy-query assoc-in [:query :filter] new-filter-clause))))))+ new-filter-clauses :- [:maybe [:sequential [:maybe ::ads/filter-clause]]]]+ (reduce (fn [query filter-clause]+ (cond-> query+ filter-clause+ (lib/filter filter-clause)))+ query+ new-filter-clauses))-(defn- inject-filter+(mu/defn- inject-filter"Inject filter clause into card."- [{:keys [query-filter cell-query] :as root} card]+ [{:keys [query-filter cell-query] :as root} :- ::ads/root+ card :- [:map+ [:dataset_query ::ads/query]]](-> card- (update :dataset_query #(add-filter-clauses % [query-filter cell-query]))+ (update :dataset_query #(add-filter-clauses % (cons cell-query query-filter)))(update :series (partial map (partial inject-filter root)))))(mu/defn- multiseries?[card :- [:map[:dataset_query {:optional true} ::ads/query]]](or (-> card :series not-empty)- (some-> card- :dataset_query- not-empty- (magic.util/do-with-legacy-query- (fn [query]- (or (-> query (get-in [:query :aggregation]) count (> 1))- (-> query (get-in [:query :breakout]) count (> 1))))))))+ (when-let [query (not-empty (:dataset_query card))]+ (or (-> query lib/aggregations count (> 1))+ (-> query lib/breakouts count (> 1))))))(mu/defn- overlay-comparison?[card :- [:map@@ -97,7 +92,7 @@card-left (->> card (inject-filter left) clone-card)card-right (->> card (inject-filter right) clone-card)[color-left color-right] (->> [left right]- (map #(magic.util/do-with-legacy-query (:dataset_query %) get-in [:query :filter]))+ (map #(some-> % :dataset_query lib/filters))populate/map-to-colors)](if (overlay-comparison? card)(let [card (-> card-left@@ -185,7 +180,7 @@[:dataset_query {:optional true} ::ads/query]]](get-in card [:visualization_settings :graph.series_labels](map (comp capitalize-first names/metric-name)- (magic.util/do-with-legacy-query (:dataset_query card) get-in [:query :aggregation]))))+ (lib/aggregations (:dataset_query card)))))(mu/defn- unroll-multiseries[card :- [:map@@ -193,10 +188,13 @@(if (and (multiseries? card)(-> card :display (= :line)))(for [[aggregation label] (map vector- (magic.util/do-with-legacy-query (:dataset_query card) get-in [:query :aggregation])+ (lib/aggregations (:dataset_query card))(series-labels card))](-> card- (update :dataset_query magic.util/do-with-legacy-query assoc-in [:query :aggregation] [aggregation])+ (update :dataset_query (fn [query]+ (-> query+ lib/remove-all-aggregations+ (lib/aggregate aggregation))))(assoc :name label)(m/dissoc-in [:visualization_settings :graph.series_labels])))[card]))@@ -248,67 +246,68 @@(and ((some-fn :cell-query :query-filter) left)(not ((some-fn :cell-query :query-filter) right))))-(defn comparison-dashboard+(mu/defn comparison-dashboard"Create a comparison dashboard based on dashboard `dashboard` comparing subsets ofthe dataset defined by segments `left` and `right`."[dashboard left right opts]- (let [left (-> left- ->root- (merge (:left opts)))- right (-> right- ->root- (merge (:right opts)))- left (cond-> left- (-> opts :left :cell-query)- (assoc :comparison-name (->> opts- :left- :cell-query- (names/cell-title left))))- right (cond-> right- (part-vs-whole-comparison? left right)- (assoc :comparison-name (condp mi/instance-of? (:entity right)- :model/Table- (tru "All {0}" (:short-name right))-- (tru "{0}, all {1}"- (comparison-name right)- (names/source-name right)))))- segment-dashboards (->> (concat (segment-constituents left)- (segment-constituents right))- distinct- (map #(automagic-analysis % {:source (:source left)- :rules-prefix ["comparison"]})))]- (assert (or (let [left-source (m/update-existing (:source left) :dataset_query magic.util/do-with-legacy-query identity)- right-source (m/update-existing (:source right) :dataset_query magic.util/do-with-legacy-query identity)]- (= left-source right-source))- (= (-> left :source :table_id)- (-> right :source u/the-id))))- (->> (concat segment-dashboards [dashboard])- (reduce (fn [dashboard-1 dashboard-2]- (if dashboard-1- (populate/merge-dashboards dashboard-1 dashboard-2 {:skip-titles? true})- dashboard-2))- nil)- dashboard->cards- (m/distinct-by (some-fn :dataset_query hash))- (transduce (mapcat unroll-multiseries)- (fn- ([]- (let [title (tru "Comparison of {0} and {1}"- (comparison-name left)- (comparison-name right))]- (-> {:name title- :transient_name title- :transient_filters nil- :param_fields nil- :description (tru "Automatically generated comparison dashboard comparing {0} and {1}"- (comparison-name left)- (comparison-name right))- :creator_id api/*current-user-id*- :parameters []- :related (update-related (:related dashboard) left right)}- (add-title-row left right))))- ([[dashboard _row]] dashboard)- ([[dashboard row] card]- [(comparison-row dashboard row left right card)- (+ row (:height card))]))))))+ (binding [lib.schema/*HACK-disable-ref-validation* true]+ (let [opts (-> opts+ (m/update-existing-in [:left :cell-query] (partial lib/normalize ::ads/root.cell-query))+ (m/update-existing-in [:right :cell-query] (partial lib/normalize ::ads/root.cell-query)))+ left (-> left+ ->root+ (merge (:left opts)))+ right (-> right+ ->root+ (merge (:right opts)))+ left (cond-> left+ (-> opts :left :cell-query)+ (assoc :comparison-name (->> opts+ :left+ :cell-query+ (names/cell-title left))))+ right (cond-> right+ (part-vs-whole-comparison? left right)+ (assoc :comparison-name (if (mi/instance-of? :model/Table (:entity right))+ (tru "All {0}" (:short-name right))+ (tru "{0}, all {1}"+ (comparison-name right)+ (names/source-name right)))))+ segment-dashboards (->> (concat (segment-constituents left)+ (segment-constituents right))+ distinct+ (map #(automagic-analysis % {:source (:source left)+ :rules-prefix ["comparison"]})))]+ (assert (or (lib.equality/= (get-in left [:source :dataset_query])+ (get-in right [:source :dataset_query]))+ (= (-> left :source :table_id)+ (-> right :source u/the-id))))+ (->> (concat segment-dashboards [dashboard])+ (reduce (fn [dashboard-1 dashboard-2]+ (if dashboard-1+ (populate/merge-dashboards dashboard-1 dashboard-2 {:skip-titles? true})+ dashboard-2))+ nil)+ dashboard->cards+ (m/distinct-by (some-fn :dataset_query hash))+ (transduce (mapcat unroll-multiseries)+ (fn+ ([]+ (let [title (tru "Comparison of {0} and {1}"+ (comparison-name left)+ (comparison-name right))]+ (-> {:name title+ :transient_name title+ :transient_filters nil+ :param_fields nil+ :description (tru "Automatically generated comparison dashboard comparing {0} and {1}"+ (comparison-name left)+ (comparison-name right))+ :creator_id api/*current-user-id*+ :parameters []+ :related (update-related (:related dashboard) left right)}+ (add-title-row left right))))+ ([[dashboard _row]] dashboard)+ ([[dashboard row] card]+ [(comparison-row dashboard row left right card)+ (+ row (:height card))])))))))diff --git a/src/metabase/xrays/automagic_dashboards/core.clj b/src/metabase/xrays/automagic_dashboards/core.cljindex 68039936194..0808f4048a7 100644--- a/src/metabase/xrays/automagic_dashboards/core.clj+++ b/src/metabase/xrays/automagic_dashboards/core.clj@@ -64,12 +64,12 @@## The Dynamic Binding and Dashboard Generation Process ##- Once data has been accreted in `automagic-analysis`, `automagic-dashboard` will first select a template as described- above. It then calls `metabase.xrays.automagic-dashboards.interesting/identify` which takes the actual column data and+ Once data has been accreted in [[automagic-analysis]], [[automagic-dashboard]] will first select a template as described+ above. It then calls [[metabase.xrays.automagic-dashboards.interesting/identify]] which takes the actual column data anddimension, metric, and filter definitions from the template and matches all potential columns to potential dimensions,- metrics, and filters. The resulting \"grounded-values\" are now passed into `generate-dashboard`, which matches all+ metrics, and filters. The resulting \"grounded-values\" are now passed into [[generate-dashboard]], which matches allof these values to card templates to produce a dashboard. The majority of the card generation work is done in- `metabase.xrays.automagic-dashboards.combination/grounded-metrics->dashcards`.+ [[metabase.xrays.automagic-dashboards.combination/grounded-metrics->dashcards]].Note that if a card template's dimensions, metrics, and filters are not matched to grounded values the card will notbe generated. Conversely, if a card template can be matched by multiple combinations of dimensions, multiple cards@@ -148,17 +148,18 @@[kixi.stats.math :as math][medley.core :as m][metabase.analyze.core :as analyze]- ;; legacy usage, do not use this in new code- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s][metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.aggregation :as lib.schema.aggregation][metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.query-processor.util :as qp.util]+ [metabase.segments.schema :as segments.schema][metabase.util :as u][metabase.util.i18n :as i18n :refer [tru trun]][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][metabase.warehouse-schema.models.field :as field][metabase.xrays.automagic-dashboards.combination :as combination]@@ -195,7 +196,7 @@:dashboard-templates-prefix ["table"]})(mu/defmethod ->root :model/Segment :- ::ads/root- [segment]+ [segment :- ::segments.schema/segment](let [table (->> segment :table_id (t2/select-one :model/Table :id))]{:entity segment:full-name (tru "{0} in the {1} segment" (:display_name table) (:name segment))@@ -203,13 +204,13 @@:comparison-name (tru "{0} segment" (:name segment)):source table:database (:db_id table)- :query-filter [:segment (u/the-id segment)]+ :query-filter [(lib/segment (u/the-id segment))]:url (format "%ssegment/%s" public-endpoint (u/the-id segment)):dashboard-templates-prefix ["table"]}))(mu/defmethod ->root :xrays/Metric :- ::ads/root- [metric :- ::ads/metric]- (let [table (some->> metric :table_id (t2/select-one :model/Table :id))]+ [{:keys [table-id], :as metric} :- ::ads/metric]+ (let [table (some->> table-id (t2/select-one :model/Table :id))]{:entity metric:full-name (if (:id metric)(trun "{0} metric" "{0} metrics" (:name metric))@@ -238,7 +239,7 @@(mu/defn- source-card-id [card-or-question :- [:map[:dataset_query ::ads/query]]]- (magic.util/do-with-mbql5-query (:dataset_query card-or-question) lib/source-card-id))+ (lib/source-card-id (:dataset_query card-or-question)))(mu/defn- nested-query?"Is this card or question derived from another model or question?"@@ -250,7 +251,7 @@"Is this card or question native (SQL)?"[{query :dataset_query, :as _card-or-question} :- [:map[:dataset_query ::ads/query]]]- (magic.util/do-with-legacy-query query #(= (:type %) :native)))+ (lib/native-only-query? query))(mu/defn- source-question :- [:maybe (ms/InstanceOf :model/Card)][card-or-question :- [:map@@ -259,14 +260,11 @@(t2/select-one :model/Card :id source-card-id)))(mu/defn- table-like?- [card-or-question :- [:map- [:dataset_query ::ads/query]]]- (magic.util/do-with-legacy-query- (:dataset_query card-or-question)- (fn [query]- (and- (nil? (get-in query [:query :aggregation]))- (nil? (get-in query [:query :breakout]))))))+ [{query :dataset_query, :as _card-or-question} :- [:map+ [:dataset_query ::ads/query]]]+ (and+ (empty? (lib/aggregations query))+ (empty? (lib/breakouts query))))(defn- table-id"Get the Table ID from `card-or-question`, which can be either a Card from the DB (which has a `:table_id` property)@@ -275,8 +273,7 @@;; TODO - probably better if we just changed `adhoc-query` to use the same keys as Cards (e.g. `:table_id`) so we;; didn't need this function, seems like something that would be too easy to forget[card-or-question]- (or (:table_id card-or-question)- (:table-id card-or-question)))+ ((some-fn :table-id :table_id) card-or-question))(mu/defn- source[card :- [:map@@ -293,17 +290,13 @@:else (->> card table-id (t2/select-one :model/Table :id))))(mu/defmethod ->root :model/Card :- ::ads/root- [card]- (let [card (update card- :dataset_query- ;; temporary until we rework X-rays to use Lib/MBQL 5- #_{:clj-kondo/ignore [:discouraged-var]}- lib/->legacy-MBQL)- source (source card)]+ [card :- [:map+ [:dataset_query ::ads/query]]]+ (let [source (source card)]{:entity card:source source:database (:database_id card)- :query-filter (magic.util/do-with-legacy-query (:dataset_query card) get-in [:query :filter])+ :query-filter (lib/filters (:dataset_query card)):full-name (tru "\"{0}\"" (:name card)):short-name (names/source-name {:source source}):url (format "%s%s/%s" public-endpoint (name (:type source :question)) (u/the-id card))@@ -314,14 +307,12 @@(mu/defmethod ->root :model/Query :- ::ads/root[query :- [:map[:database-id ::lib.schema.id/database]]]- ;; use of [[lib/->legacy-MBQL]] here is temporary until we update X-Rays to use Lib- (let [query (update query :dataset_query #_{:clj-kondo/ignore [:discouraged-var]} lib/->legacy-MBQL)- source (source query)+ (let [source (source query)root {:database (:database-id query), :source source}]{:entity query:source source:database (:database-id query)- :query-filter (magic.util/do-with-legacy-query (:dataset_query query) get-in [:query :filter])+ :query-filter (lib/filters (:dataset_query query)):full-name (cond(native-query? query) (tru "Native query")(table-like? query) (-> source ->root :full-name)@@ -380,18 +371,19 @@x)(m/update-existing :visualization #(instantiate-visualization % bindings available-metrics))))-(defn- singular-cell-dimension-field-ids+(mu/defn- singular-cell-dimension-field-ids :- [:maybe [:set [:or :string ::lib.schema.id/field]]]"Return the set of ids referenced in a cell query"- [{:keys [cell-query]}]- (letfn [(collect-dimensions [[op & args]]- (case (some-> op qp.util/normalize-token)- :and (mapcat collect-dimensions args)- := (magic.util/collect-field-references args)+ [{:keys [cell-query], :as _root} :- ::ads/root]+ (letfn [(collect-dimensions [[tag _opts & args]]+ ;; TODO (Cam 10/21/25) -- piccing apart MBQL clauses like this is a little icky and unidiomatic, we really+ ;; don't discourage digging around in MBQL outside of Lib -- FIXME+ (case (some-> tag keyword)+ :and (mapcat collect-dimensions args)+ (:between :=) (magic.util/collect-field-references args)nil))]- (->> cell-query- collect-dimensions- (map magic.util/field-reference->id)- set)))+ (into #{}+ (map magic.util/field-reference->id)+ (collect-dimensions cell-query))))(defn- matching-dashboard-templates"Return matching dashboard templates ordered by specificity.@@ -467,8 +459,7 @@{:source (assoc source :fields (table->fields source)):root root:tables (map #(assoc % :fields (table->fields %)) tables)- :query-filter (filters/inject-refinement (:query-filter root)- (:cell-query root))}))+ :query-filter (filters/inject-refinement (:query-filter root) (:cell-query root))}))(defn- make-dashboard([root dashboard-template]@@ -493,7 +484,7 @@:as dashboard-template}{grounded-dimensions :dimensionsgrounded-metrics :metrics- grounded-filters :filters}]+ grounded-filters :filters} :- ::ads/grounded-values](let [card-templates (interesting/normalize-seq-of-maps :card template-cards)dashcards (combination/grounded-metrics->dashcardsbase-context@@ -695,7 +686,7 @@"Produce a fully-populated dashboard from the base context for an item and a dashboard template."[{{:keys [show url query-filter] :as root} :root :as base-context} :- ::ads/context{:as dashboard-template}- {grounded-dimensions :dimensions :as grounded-values}]+ {grounded-dimensions :dimensions :as grounded-values} :- ::ads/grounded-values](let [show (or show max-cards)dashboard (generate-base-dashboard base-context dashboard-template grounded-values)](-> dashboard@@ -731,7 +722,33 @@:filter-specs template-filters})](generate-dashboard base-context template grounded-values)))-(defmulti automagic-analysis+(mr/def ::automagic-analysis.opts+ [:maybe+ [:map+ {:closed true}+ [:cell-query {:optional true} [:maybe ::ads/root.cell-query]]+ [:show {:optional true} [:maybe [:or pos-int? [:= :all]]]]+ [:source {:optional true} ::ads/source]+ [:query-filter {:optional true} [:maybe+ [:schema+ {:decode/normalize (fn [filters]+ (when (seq filters)+ (if (sequential? (first filters))+ filters+ [filters])))}+ [:sequential ::ads/filter-clause]]]]+ [:database {:optional true} ::lib.schema.id/database]+ [:comparison? {:optional true} [:maybe :boolean]]+ [:rules-prefix {:optional true} [:maybe [:sequential :string]]]+ ;; TODO -- use [[metabase.xrays.api.automagic-dashboards/DashboardTemplate]] (move it somewhere first)+ [:dashboard-template {:optional true} [:maybe [:sequential :string]]]]])++(defmulti ^:private automagic-analysis-method+ {:arglists '([entity opts])}+ (fn [entity _]+ (mi/model entity)))++(defn automagic-analysis"Create a transient dashboard analyzing given entity.This function eventually calls out to `automagic-dashboard` with two primary arguments:@@ -739,46 +756,47 @@passed through the `->root` function, which is an aggregate including the original entity, itssource, what dashboard template categories to apply, etc.- Additional options such as how many cards to show, a cell query (a drill through), etc."- {:arglists '([entity opts])}- (fn [entity _]- (mi/model entity)))+ [entity opts]+ (automagic-analysis-method+ (lib/normalize ::ads/root.entity entity)+ (lib/normalize ::automagic-analysis.opts opts)))-(defmethod automagic-analysis :model/Table+(defmethod automagic-analysis-method :model/Table[table opts](automagic-dashboard (merge (->root table) opts)))-(defmethod automagic-analysis :model/Segment- [segment opts]+(mu/defmethod automagic-analysis-method :model/Segment+ [segment :- ::segments.schema/segment+ opts](automagic-dashboard (merge (->root segment) opts)))-(mu/defmethod automagic-analysis :xrays/Metric+(mu/defmethod automagic-analysis-method :xrays/Metric[metric :- ::ads/metricopts](automagic-dashboard (merge (->root metric) opts)))-(mu/defn- collect-metrics :- [:maybe [:sequential (ms/InstanceOf :xrays/Metric)]]+(mu/defn- collect-metrics :- [:maybe [:sequential [:and+ (ms/InstanceOf :xrays/Metric)+ ::ads/metric]]][root :- ::ads/rootquestion :- [:map[:dataset_query ::ads/query]]]- (map (mu/fn [aggregation-clause :- ::mbql.s/Aggregation]- (if (-> aggregation-clause- first- qp.util/normalize-token- (= :metric))+ (map (mu/fn [aggregation-clause :- ::lib.schema.aggregation/aggregation]+ (if (lib/clause-of-type? aggregation-clause :metric);; any [:metric ...] MBQL clauses these days are V2 Metrics and Automagic Dashboards do not handle them.(log/error "X-Rays do not support V2 Metrics.")(let [table-id (table-id question)]- (mi/instance :xrays/Metric {:definition {:aggregation [aggregation-clause]- :source-table table-id}- :name (names/metric->description root aggregation-clause)- :table_id table-id}))))- (magic.util/do-with-legacy-query (:dataset_query question) get-in [:query :aggregation])))+ (mi/instance :xrays/Metric {:xrays/aggregation aggregation-clause+ :name (names/metric->description root aggregation-clause)+ :table-id table-id+ :xrays/database-id ((some-fn :database-id :database_id) question)}))))+ (lib/aggregations (:dataset_query question))))(mu/defn- collect-breakout-fields :- [:maybe [:sequential (ms/InstanceOf :model/Field)]][root :- ::ads/rootquestion :- [:map[:dataset_query ::ads/query]]]- (for [breakout (magic.util/do-with-legacy-query (:dataset_query question) get-in [:query :breakout])+ (for [breakout (lib/breakouts (:dataset_query question))field-clause (take 1 (magic.util/collect-field-references breakout)):let [field (magic.util/->field root field-clause)]:when (and field@@ -807,23 +825,41 @@[(collect-metrics root question)(collect-breakout-fields root question)])))-(defn- preserve-entity-element+(mu/defn- preserve-entity-element"Ensure that elements of an original dataset query are preserved in dashcard queries."[dashboardentity- entity-element]- (if-let [element-value (magic.util/do-with-legacy-query (:dataset_query entity) get-in [:query entity-element])]- (letfn [(splice-element [dashcard]- (m/update-existing-in dashcard [:card :dataset_query]- magic.util/do-with-legacy-query- update-in [:query entity-element]- (fnil into (empty element-value))- element-value))]- (update dashboard :dashcards (partial map splice-element)))- dashboard))--(defn- query-based-analysis- [{:keys [entity] :as root} opts {:keys [cell-query cell-url]}]+ getter-fn+ setter-fn]+ (binding [lib.schema/*HACK-disable-ref-validation* true]+ (if-let [element-value (some-> entity :dataset_query not-empty getter-fn)]+ (letfn [(splice-element [dashcard]+ (m/update-existing-in dashcard [:card :dataset_query] setter-fn element-value))]+ (update dashboard :dashcards (partial mapv splice-element)))+ dashboard)))++(defn- preserve-joins [dashboard entity]+ (preserve-entity-element dashboard entity lib/joins (fn [query joins]+ (reduce lib/join query joins))))++(defn- preserve-expressions [dashboard entity]+ (preserve-entity-element dashboard entity lib/expressions (fn [query expressions]+ (reduce+ (fn [query expression]+ (lib/expression query+ (:lib/expression-name (lib/options expression))+ expression))+ query+ expressions))))++(mu/defn- query-based-analysis+ [{:keys [entity] :as root} :- ::ads/root+ opts :- ::automagic-analysis.opts+ {:keys [cell-query cell-url]} :- [:maybe+ [:map+ {:closed true}+ [:cell-query {:optional true} [:maybe ::ads/root.cell-query]]+ [:cell-url :string]]]](let [transient-dash (if (table-like? entity)(let [root' (merge root(when cell-query@@ -847,11 +883,12 @@{:transient_name title:name title})))))](-> transient-dash- (preserve-entity-element (:entity root) :joins)- (preserve-entity-element (:entity root) :expressions))))+ (preserve-joins (:entity root))+ (preserve-expressions (:entity root)))))-(defmethod automagic-analysis :model/Card- [card {:keys [cell-query] :as opts}]+(mu/defmethod automagic-analysis-method :model/Card+ [card+ {:keys [cell-query] :as opts} :- ::automagic-analysis.opts](let [root (->root card)cell-url (format "%squestion/%s/cell/%s" public-endpoint(u/the-id card)@@ -861,25 +898,22 @@{:cell-query cell-query:cell-url cell-url}))))-(defmethod automagic-analysis :model/Query- [query {:keys [cell-query] :as opts}]- (let [root (->root query)- cell-query (when cell-query- #_{:clj-kondo/ignore [:deprecated-var]}- (mbql.normalize/normalize-fragment [:query :filter] cell-query))- opts (cond-> opts- cell-query (assoc :cell-query cell-query))- cell-url (format "%sadhoc/%s/cell/%s" public-endpoint- (magic.util/encode-base64-json (:dataset_query query))- (magic.util/encode-base64-json cell-query))]- (query-based-analysis root opts- (when cell-query- {:cell-query cell-query- :cell-url cell-url}))))+(mu/defmethod automagic-analysis-method :model/Query+ [query+ {:keys [cell-query] :as opts} :- ::automagic-analysis.opts]+ (let [root (->root query)+ cell-url (format "%sadhoc/%s/cell/%s" public-endpoint+ (magic.util/encode-base64-json (:dataset_query query))+ (magic.util/encode-base64-json cell-query))]+ (binding [lib.schema/*HACK-disable-ref-validation* true]+ (query-based-analysis root opts+ (when cell-query+ {:cell-query cell-query+ :cell-url cell-url})))))-(mu/defmethod automagic-analysis :model/Field+(mu/defmethod automagic-analysis-method :model/Field[field :- ::ads/field- opts]+ opts :- ::automagic-analysis.opts](automagic-dashboard (merge (->root field) opts)))(defn- load-tables-with-enhanced-table-statsdiff --git a/src/metabase/xrays/automagic_dashboards/filters.clj b/src/metabase/xrays/automagic_dashboards/filters.cljindex e498511e9ed..a9e12855320 100644--- a/src/metabase/xrays/automagic_dashboards/filters.clj+++ b/src/metabase/xrays/automagic_dashboards/filters.clj@@ -1,8 +1,8 @@(ns metabase.xrays.automagic-dashboards.filters(:require- ;; legacy usage, do not use this in new code- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]- ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.legacy-mbql.util :as mbql.u]+ [metabase.lib.core :as lib]+ [metabase.lib.equality :as lib.equality]+ [metabase.lib.schema :as lib.schema][metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.malli :as mu]@@ -121,7 +121,8 @@"Add up to `max-filters` filters to dashboard `dashboard`. The `dimensions` argument is a list of fields for which tocreate filters."[dashboard :- ::ads/dashboard- dimensions max-filters]+ dimensions+ max-filters](let [fks (when-let [table-ids (not-empty (set (keep (comp :table_id :card)(:dashcards dashboard))))](field/with-targets (t2/select :model/Field@@ -149,25 +150,7 @@dashboard)))dashboard))))-(defn- flatten-filter-clause- "Returns a sequence of filter subclauses making up `filter-clause` by flattening `:and` compound filters.-- (flatten-filter-clause [:and- [:= [:field 1 nil] 2]- [:and- [:= [:field 3 nil] 4]- [:= [:field 5 nil] 6]]])- ;; -> ([:= [:field 1 nil] 2]- [:= [:field 3 nil] 4]- [:= [:field 5 nil] 6])"- [[clause-name, :as filter-clause]]- (when (seq filter-clause)- (if (= clause-name :and)- #_{:clj-kondo/ignore [:deprecated-var]}- (rest (mbql.u/simplify-compound-filter filter-clause))- [filter-clause])))--(defn inject-refinement+(mu/defn inject-refinement :- [:maybe ::lib.schema/filters]"Inject a filter refinement into an MBQL filter clause, returning a new filter clause.There are two reasons why we want to do this: 1) to reduce visual noise when we display applied filters; and 2) some@@ -177,17 +160,19 @@of the latter. Therefore we can rewrite the combined clause to ommit the more broad version from the main clause.Assumes both filter clauses can be flattened by recursively merging `:and` claueses(ie. no `:and`s inside `:or` or `:not`)."- [filter-clause refinement]- (let [in-refinement? (into #{}- (map magic.util/collect-field-references)- (flatten-filter-clause refinement))- existing-filters (->> filter-clause- flatten-filter-clause- (remove (comp in-refinement? magic.util/collect-field-references)))]- (if (seq existing-filters)- ;; since the filters are programatically generated they won't have passed thru normalization, so make sure we- ;; normalize them before passing them to `combine-filter-clauses`, which validates its input- #_{:clj-kondo/ignore [:deprecated-var]}- (apply mbql.u/combine-filter-clauses (map (partial mbql.normalize/normalize-fragment [:query :filter])- (cons refinement existing-filters)))- refinement)))+ [filter-clauses :- [:maybe ::lib.schema/filters]+ refinement :- [:maybe vector?]]+ (if (some? refinement)+ ;; normalize refinement since it's read from the YAML files or whatever+ (let [refinement (lib/normalize refinement)+ ;; TODO (Cam 10/21/2025) -- HACK -- I wanted to use [[lib/filter-parts]] for this but we need to pass in+ ;; query as part of this. Maybe we can refactor this code so we can use that, or just add a new function to+ ;; see if we have existing filters against a column.+ refinement-column (nth refinement 2)+ ;; remove any existing filters against the column in the refinement filter.+ filter-clauses' (into []+ (remove (fn [a-filter]+ (lib.equality/= (nth a-filter 2) refinement-column)))+ (lib/simplify-filters filter-clauses))]+ (lib/simplify-filters (conj filter-clauses' refinement)))+ filter-clauses))diff --git a/src/metabase/xrays/automagic_dashboards/interesting.clj b/src/metabase/xrays/automagic_dashboards/interesting.cljindex f6e0d771767..26a971e4759 100644--- a/src/metabase/xrays/automagic_dashboards/interesting.clj+++ b/src/metabase/xrays/automagic_dashboards/interesting.clj@@ -41,13 +41,11 @@(:require[clojure.math.combinatorics :as math.combo][clojure.string :as str]- [clojure.walk :as walk][java-time.api :as t][medley.core :as m]- ;; legacy usages, do not use legacy MBQL stuff in new code.- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]- ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.legacy-mbql.util :as mbql.u]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.ref :as lib.schema.ref][metabase.models.interface :as mi][metabase.util :as u]@@ -91,27 +89,28 @@(can-use? :hour) :hour))(if (can-use? :day) :day :hour))))-(mu/defmethod ->reference [:mbql :model/Field] :- ::mbql.s/field- [_ {:keys [fk_target_field_id id link aggregation name base_type] :as field}]- (let [reference (mbql.normalize/normalize- (cond- link [:field id {:source-field link}]- fk_target_field_id [:field fk_target_field_id {:source-field id}]- id [:field id {:base-type base_type}]- :else [:field name {:base-type base_type}]))]- ;; legacy usages -- use Lib in new code going forward- #_{:clj-kondo/ignore [:deprecated-var]}- (cond- (isa? base_type :type/Temporal)- (mbql.u/with-temporal-unit reference (keyword (or aggregation- (optimal-temporal-resolution field))))+(mu/defn field->metadata :- ::lib.schema.metadata/column+ "Convert a Field from the app DB to Lib column metadata, and do a bunch of weird additional transformations that+ I (Cam) do not really understand (see below).++ TODO (Cam 10/21/25) -- we wouldn't need to do this at all if we further reworked the X-Ray code to either fetch+ `:metadata/column`s directly or use Metadata Providers."+ [{fk-target-field-id :fk_target_field_id, base-type :base_type, :keys [id link aggregation], :as field} :- (ms/InstanceOf :model/Field)]+ (let [col (if fk-target-field-id+ (-> (t2/select-one :metadata/column :id fk-target-field-id)+ (assoc :fk-field-id id, :lib/source :source/implicitly-joinable))+ (lib-be/instance->metadata field :metadata/column))]+ (cond-> col+ link+ (assoc :fk-field-id link, :lib/source :source/implicitly-joinable)++ (isa? base-type :type/Temporal)+ (lib/with-temporal-bucket (keyword (or aggregation+ (optimal-temporal-resolution field))))(and aggregation- (isa? base_type :type/Number))- (mbql.u/update-field-options reference assoc-in [:binning :strategy] (keyword aggregation))-- :else- reference)))+ (isa? base-type :type/Number))+ (lib/with-binning {:strategy (keyword aggregation)}))))(defmethod ->reference [:string :model/Field][_ {:keys [display_name full-name link]}]@@ -131,10 +130,10 @@(or full-name name))(defmethod ->reference [:mbql :xrays/Metric]- [_ {:keys [id definition]}]+ [_ {:keys [id], :as metric}](if id[:metric id]- (-> definition :aggregation first)))+ (:xrays/aggregation metric)))(defmethod ->reference [:native :model/Field][_ field]@@ -150,44 +149,43 @@(map? form) ((some-fn :full-name :name) form))form))-(defn transform-metric-aggregate+(mu/defn transform-metric-aggregate :- ::ads/external-op"Map a metric aggregate definition from nominal types to semantic types."- [m decoder]- (walk/prewalk- (fn [v]- (if (vector? v)- (let [[d n] v]- (if (= "dimension" d)- (decoder n)- v))- v))- m))--(mu/defn ground-metric :- [:sequential ads/grounded-metric]+ [[ag-type & args] dimension-name->field]+ {:lib/type :lib/external-op+ :operator (keyword ag-type)+ :args (mapv (fn [arg]+ (if (and (vector? arg)+ (= (first arg) "dimension"))+ (when-let [field (dimension-name->field (second arg))]+ (field->metadata field))+ arg))+ args)})++(mu/defn ground-metric :- [:sequential ::ads/grounded-metric]"Generate \"grounded\" metrics from the mapped dimensions (dimension name -> field matches).Since there may be multiple matches to a dimension, this will produce a sequence of potential matches."[{metric-name :metric-namemetric-score :score- metric-definition :metric} :- ads/normalized-metric-template- ground-dimensions :- ads/dim-name->matching-fields]+ metric-definition :metric} :- ::ads/normalized-metric-template+ ground-dimensions :- ::ads/dim-name->matching-fields](let [named-dimensions (dashboard-templates/collect-dimensions metric-definition)](->> (map (comp :matches ground-dimensions) named-dimensions)(apply math.combo/cartesian-product)(map (partial zipmap named-dimensions))(map (fn [nm->field]- (let [xform (update-vals nm->field (partial ->reference :mbql))]- {:metric-name metric-name- :metric-title metric-name- :metric-score metric-score- :metric-definition {:aggregation- [(transform-metric-aggregate metric-definition xform)]}- ;; Required for title interpolation in grounded-metrics->dashcards- :dimension-name->field nm->field}))))))--(mu/defn grounded-metrics :- [:sequential ads/grounded-metric]+ {:metric-name metric-name+ :metric-title metric-name+ :metric-score metric-score+ :metric-definition {:xrays/aggregations+ [(transform-metric-aggregate metric-definition nm->field)]}+ ;; Required for title interpolation in grounded-metrics->dashcards+ :dimension-name->field nm->field})))))++(mu/defn grounded-metrics :- [:sequential ::ads/grounded-metric]"Given a set of metric definitions and grounded (assigned) dimensions, produce a sequence of grounded metrics."- [metric-templates :- [:sequential ads/normalized-metric-template]- ground-dimensions :- ads/dim-name->matching-fields]+ [metric-templates :- [:sequential ::ads/normalized-metric-template]+ ground-dimensions :- ::ads/dim-name->matching-fields](mapcat #(ground-metric % ground-dimensions) metric-templates))(defn normalize-seq-of-maps@@ -342,7 +340,7 @@(let [scored-bindings (score-bindings candidate-binding-values)](second (last (sort-by first scored-bindings)))))-(mu/defn find-dimensions :- ads/dim-name->dim-defs+matches+(mu/defn find-dimensions :- ::ads/dim-name->dim-defs+matches"Bind fields to dimensions from the dashboard template and resolve overloaded cases in which multiple fields match thedimension specification.@@ -351,7 +349,7 @@(see `most-specific-definition` for details).The context is passed in, but it only needs tables and fields in `candidate-bindings`. It is not extensively used."- [context dimension-specs :- [:maybe [:sequential ads/dimension-template]]]+ [context dimension-specs :- [:maybe [:sequential ::ads/dimension-template]]](->> (candidate-bindings context dimension-specs)(map (comp most-specific-matched-dimension val))(apply merge-with (fn [a b]@@ -392,7 +390,7 @@(add-field-links-to-definitions (enriched-field-with-sources context entity))));;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(defn grounded-filters+(mu/defn grounded-filters :- [:sequential ::ads/grounded-filter]"Take filter templates (as from a dashboard template's :filters) and ground dimensions and produce a map of thefilter name to grounded versions of the filter."[filter-templates ground-dimensions]@@ -401,40 +399,36 @@(let [[fname {:keys [filter] :as v}] (first fltr)dims (dashboard-templates/collect-dimensions v)opts (->> (map (comp- (partial map (partial ->reference :mbql)):matchesground-dimensions) dims)(apply math.combo/cartesian-product)(map (partial zipmap dims)))]- (seq (for [opt opts- :let [f- (walk/prewalk- (fn [x]- (if (vector? x)- (let [[ds dim-name] x]- (if (and (= "dimension" ds)- (string? dim-name))- (opt dim-name)- x))- x))- filter)]]+ (seq (for [opt opts+ :let [[op & args] filter+ f+ {:lib/type :lib/external-op+ :operator (keyword op)+ :args (mapv (fn [arg]+ (if (and (vector? arg)+ (= (first arg) "dimension"))+ (when-let [field (opt (second arg))]+ (field->metadata field))+ arg))+ args)}]](assoc v :filter f :filter-name fname))))))flatten))-(mu/defn identify- :- [:map- [:dimensions ads/dim-name->matching-fields]- [:metrics [:sequential ads/grounded-metric]]]+(mu/defn identify :- ::ads/grounded-values"Identify interesting metrics and dimensions of a `thing`. First identifies interesting dimensions, and theninteresting metrics which are satisfied.Metrics from the template are assigned a score of 50; user defined metrics a score of 95"- [context+ [context :- ::ads/context{:keys [dimension-specsmetric-specsfilter-specs]} :- [:map- [:dimension-specs [:maybe [:sequential ads/dimension-template]]]- [:metric-specs [:maybe [:sequential ads/metric-template]]]- [:filter-specs [:maybe [:sequential ads/filter-template]]]]]+ [:dimension-specs [:maybe [:sequential ::ads/dimension-template]]]+ [:metric-specs [:maybe [:sequential ::ads/metric-template]]]+ [:filter-specs [:maybe [:sequential ::ads/filter-template]]]]](let [dims (->> (find-dimensions context dimension-specs)(add-field-self-reference context))metrics (-> (normalize-seq-of-maps :metric metric-specs)@@ -448,6 +442,6 @@(when (mi/instance-of? :xrays/Metric entity)[{:metric-name "this":metric-title (:name entity)- :metric-definition {:aggregation [(->reference :mbql entity)]}+ :metric-definition {:xrays/aggregations [(:xrays/aggregation entity)]}:metric-score dashboard-templates/max-score}]))):filters (grounded-filters filter-specs dims)}))diff --git a/src/metabase/xrays/automagic_dashboards/names.clj b/src/metabase/xrays/automagic_dashboards/names.cljindex 7ddce36d57f..db2ac742451 100644--- a/src/metabase/xrays/automagic_dashboards/names.clj+++ b/src/metabase/xrays/automagic_dashboards/names.clj@@ -2,12 +2,12 @@(:require[clojure.string :as str][java-time.api :as t]- ;; legacy usages, do not use legacy MBQL stuff in new code.- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]- [metabase.lib.util.match :as lib.util.match]- [metabase.query-processor.util :as qp.util]+ [metabase.lib.core :as lib]+ [metabase.lib.schema.aggregation :as lib.schema.aggregation]+ [metabase.lib.schema.expression :as lib.schema.expression]+ [metabase.lib.schema.ref :as lib.schema.ref][metabase.util.date-2 :as u.date]- [metabase.util.i18n :refer [deferred-tru tru]]+ [metabase.util.i18n :as i18n :refer [deferred-tru tru]][metabase.util.malli :as mu][metabase.util.time :as u.time][metabase.xrays.automagic-dashboards.schema :as ads]@@ -26,36 +26,46 @@:cum-count (deferred-tru "cumulative count"):cum-sum (deferred-tru "cumulative sum")})-(defn metric-name+(mu/defn metric-name :- ::ads/string-or-18n-string"Return the name of the metric or name by describing it."- [[op & args :as metric]]- (if (magic.util/adhoc-metric? metric)- (-> op qp.util/normalize-token op->name)- (second args)))+ [[tag :as metric] :- vector?]+ (cond+ (magic.util/adhoc-metric? metric)+ (-> tag keyword op->name)-(defn- join-enumeration+ (lib/clause-of-type? metric :field)+ (lib/field-ref-name metric)++ :else+ (throw (ex-info (format "Don't know how to get the name of %s" (pr-str metric))+ {:metric metric}))))++(mu/defn- join-enumeration :- ::ads/string-or-18n-string"Join a sequence as [1 2 3 4] to \"1, 2, 3 and 4\""- [xs]+ [xs :- [:sequential :any]](if (next xs)(tru "{0} and {1}" (str/join ", " (butlast xs)) (last xs))- (first xs)))+ (str (first xs))))(def ^{:arglists '([root])} source-name"Return the (display) name of the source of a given root object."(comp (some-fn :display_name :name) :source))-(defn metric->description+(mu/defn metric->description :- [:or :string [:fn {:error/message "localized string"} i18n/localized-string?]]"Return a description for the metric."- [root aggregation-clause]+ [root :- ::ads/root+ aggregation-clause :- [:or+ ::lib.schema.aggregation/aggregation+ ::lib.schema.aggregation/aggregations]](join-enumeration(for [metric (if (sequential? (first aggregation-clause))aggregation-clause[aggregation-clause])](if (magic.util/adhoc-metric? metric)- (tru "{0} of {1}" (metric-name metric) (or (some->> metric- second- (magic.util/->field root)- :display_name)+ (tru "{0} of {1}" (metric-name metric) (or (when (> (count metric) 2)+ (->> (nth metric 2) ; icky+ (magic.util/->field root)+ :display_name))(source-name root)))(metric-name metric)))))@@ -64,12 +74,15 @@[root :- ::ads/rootquestion :- [:map[:dataset_query ::ads/query]]]- (let [aggregations (->> (magic.util/do-with-legacy-query (:dataset_query question) get-in [:query :aggregation])+ (let [aggregations (->> (lib/aggregations (:dataset_query question))(metric->description root))- dimensions (->> (magic.util/do-with-legacy-query (:dataset_query question) get-in [:query :breakout])- (mapcat magic.util/collect-field-references)- (map (partial magic.util/->field root))- (map :display_name)+ field-ids (into+ #{}+ (mapcat lib/all-field-ids)+ (lib/breakouts (:dataset_query question)))+ dimensions (->> field-ids+ (mapv (partial magic.util/->field root))+ (mapv :display_name)join-enumeration)](if dimensions(tru "{0} by {1}" aggregations dimensions)@@ -77,8 +90,8 @@(defmulti ^:private humanize-filter-value{:arglists '([root mbql-clause])}- (fn [_root [tag]]- (qp.util/normalize-token tag)))+ (fn [_root mbql-clause]+ (lib/dispatch-value mbql-clause)))(def ^:private unit-name (comp {:minute-of-hour (deferred-tru "minute"):hour-of-day (deferred-tru "hour")@@ -89,28 +102,23 @@:month-of-year (deferred-tru "month"):quarter-of-year (deferred-tru "quarter"):year (deferred-tru "year")}- qp.util/normalize-token))+ keyword))-(mu/defn- item-reference->field+(mu/defn- item-reference->field :- [:map+ [:item-name :string]]"Turn a field reference into a field."- [root :- ::ads/root- [item-type :as item-reference]]- (case item-type- (:field "field") (let [normalized-field-reference (mbql.normalize/normalize item-reference)- temporal-unit (lib.util.match/match-one normalized-field-reference- [:field _ (opts :guard :temporal-unit)]- (:temporal-unit opts))- {:keys [display_name] :as field-record} (cond-> (->> normalized-field-reference- magic.util/collect-field-references- first- (magic.util/->field root))- temporal-unit- (assoc :unit temporal-unit))- item-name (cond->> display_name- (some-> temporal-unit u.date/extract-units)- (tru "{0} of {1}" (unit-name temporal-unit)))]- (assoc field-record :item-name item-name))- (:expression "expression") {:item-name (second item-reference)}+ [root :- ::ads/root+ [tag _opts x :as a-ref] :- ::lib.schema.ref/ref]+ (case tag+ :field (let [temporal-unit (lib/raw-temporal-bucket a-ref)+ {:keys [display_name] :as field-record} (cond-> (magic.util/->field root a-ref)+ temporal-unit+ (assoc :unit temporal-unit))+ item-name (cond->> display_name+ (some-> temporal-unit u.date/extract-units)+ (tru "{0} of {1}" (unit-name temporal-unit)))]+ (assoc field-record :item-name item-name))+ :expression {:item-name x}{:item-name "item"}))(defn item-name@@ -163,48 +171,48 @@:week-of-year) (u.date/extract dt unit))))(mu/defmethod humanize-filter-value :=- [root :- ::ads/root- [_ field-reference value]]+ [root :- ::ads/root+ [_ _opts field-reference value] :- :mbql.clause/=](let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)](if (isa? (or effective_type base_type) :type/Temporal)(tru "{0} is {1}" item-name (humanize-datetime value unit))(tru "{0} is {1}" item-name value))))(mu/defmethod humanize-filter-value :>=- [root :- ::ads/root- [_ field-reference value]]+ [root :- ::ads/root+ [_ _opts field-reference value] :- :mbql.clause/>=](let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)](if (isa? (or effective_type base_type) :type/Temporal)(tru "{0} is not before {1}" item-name (humanize-datetime value unit))(tru "{0} is at least {1}" item-name value))))(mu/defmethod humanize-filter-value :>- [root :- ::ads/root- [_ field-reference value]]+ [root :- ::ads/root+ [_ _opts field-reference value] :- :mbql.clause/>](let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)](if (isa? (or effective_type base_type) :type/Temporal)(tru "{0} is after {1}" item-name (humanize-datetime value unit))(tru "{0} is greater than {1}" item-name value))))(mu/defmethod humanize-filter-value :<=- [root :- ::ads/root- [_ field-reference value]]+ [root :- ::ads/root+ [_ _opts field-reference value] :- :mbql.clause/<=](let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)](if (isa? (or effective_type base_type) :type/Temporal)(tru "{0} is not after {1}" item-name (humanize-datetime value unit))(tru "{0} is no more than {1}" item-name value))))(mu/defmethod humanize-filter-value :<- [root :- ::ads/root- [_ field-reference value]]+ [root :- ::ads/root+ [_ _opts field-reference value] :- :mbql.clause/<](let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)](if (isa? (or effective_type base_type) :type/Temporal)(tru "{0} is before {1}" item-name (humanize-datetime value unit))(tru "{0} is less than {1}" item-name value))))(mu/defmethod humanize-filter-value :between- [root :- ::ads/root- [_ field-reference min-value max-value]]+ [root :- ::ads/root+ [_ _opts field-reference min-value max-value] :- :mbql.clause/between](let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)](or (when (isa? (or effective_type base_type) :type/Temporal)(let [humanized-min (humanize-datetime min-value unit)@@ -214,32 +222,33 @@(tru "{0} is between {1} and {2}" item-name min-value max-value))))(mu/defmethod humanize-filter-value :inside- [root :- ::ads/root- [_ lat-reference lon-reference lat-max lon-min lat-min lon-max]]+ [root :- ::ads/root+ [_ _opts lat-reference lon-reference lat-max lon-min lat-min lon-max] :- :mbql.clause/inside](tru "{0} is between {1} and {2}; and {3} is between {4} and {5}"(item-name root lon-reference) lon-min lon-max(item-name root lat-reference) lat-min lat-max))(mu/defmethod humanize-filter-value :and- [root :- ::ads/root- [_ & clauses]]+ [root :- ::ads/root+ [_ _opts & clauses] :- :mbql.clause/and](->> clauses(map (partial humanize-filter-value root))join-enumeration))(mu/defmethod humanize-filter-value :default- [root :- ::ads/root- [_ field-reference value]]+ [root :- ::ads/root+ [_ field-reference value] :- ::lib.schema.expression/boolean](let [{:keys [item-name effective_type base_type unit]} (item-reference->field root field-reference)](if (isa? (or effective_type base_type) :type/Temporal)(tru "{0} relates to {1}" item-name (humanize-datetime value unit))(tru "{0} relates to {1}" item-name value))))-(defn cell-title+(mu/defn cell-title :- :string"Return a cell title given a root object and a cell query."- [root cell-query]+ [root :- ::ads/root+ cell-query :- ::ads/root.cell-query](str/join " " [(if-let [aggregation (-> (get-in root [:entity :dataset_query])- (magic.util/do-with-legacy-query get-in [:query :aggregation]))]+ lib/aggregations)](metric->description root aggregation)(:full-name root))(tru "where {0}" (humanize-filter-value root cell-query))]))diff --git a/src/metabase/xrays/automagic_dashboards/populate.clj b/src/metabase/xrays/automagic_dashboards/populate.cljindex ffe69de6767..4fb6e62932b 100644--- a/src/metabase/xrays/automagic_dashboards/populate.clj+++ b/src/metabase/xrays/automagic_dashboards/populate.clj@@ -140,7 +140,7 @@{query :dataset_query, :keys [title description width height id] :as dashcard} :- ::ads/card-template[x y]](let [query-fields (when query- (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]+ (binding [lib.schema/*HACK-disable-ref-validation* true](-> {:dataset_query (lib-be/normalize-query query)}queries/populate-card-query-fields(select-keys [:query_type :database_id :table_id]))))@@ -320,7 +320,7 @@(mu/defn- create-dashboard-populate-dashcards :- [:sequential ::ads/dashcard][dashcards :- [:maybe [:sequential ::ads/card-template]]]- (let [card-id->can-run-adhoc-query (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]+ (let [card-id->can-run-adhoc-query (binding [lib.schema/*HACK-disable-ref-validation* true](into {}(map (juxt ::id :can_run_adhoc_query))(queries/with-can-run-adhoc-querydiff --git a/src/metabase/xrays/automagic_dashboards/schema.clj b/src/metabase/xrays/automagic_dashboards/schema.cljindex b7a58d6b0ee..36bdd185f76 100644--- a/src/metabase/xrays/automagic_dashboards/schema.clj+++ b/src/metabase/xrays/automagic_dashboards/schema.clj@@ -1,19 +1,48 @@(ns metabase.xrays.automagic-dashboards.schema(:require[malli.core :as mc]- [malli.util :as mut]- ;; legacy usages, do not use legacy MBQL stuff in new code.- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s][metabase.lib.core :as lib][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.aggregation :as lib.schema.aggregation][metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]+ [metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.util.i18n :as i18n][metabase.util.malli.registry :as mr]- [metabase.util.malli.schema :as ms]))+ [metabase.util.malli.schema :as ms]+ [toucan2.core :as t2]))++(mr/def ::string-or-18n-string+ [:or :string [:fn {:error/message "localized string"} i18n/localized-string?]])++(mr/def ::root.entity+ [:multi+ {:dispatch t2/model}+ [:xrays/Metric [:ref ::metric]]+ [::mc/default [:map+ [:name {:optional true} :string]]]])++(mr/def ::filter-clause+ [:and+ {:decode/normalize (fn [x]+ (when (sequential? x)+ (if (map? (second x))+ x+ (lib/->pMBQL x))))}+ [:ref ::lib.schema.mbql-clause/clause]+ [:ref ::lib.schema.expression/boolean]])++(mr/def ::root.cell-query+ ::filter-clause)(mr/def ::root[:map- [:database ::lib.schema.id/database]])+ [:database ::lib.schema.id/database]+ [:entity {:optional true} [:ref ::root.entity]]+ [:query-filter {:optional true} [:maybe [:sequential ::filter-clause]]]+ [:cell-query {:optional true} [:maybe [:ref ::root.cell-query]]]])(mr/def ::source[:or@@ -27,290 +56,180 @@(mr/def ::context"The big ball of mud data object from which we generate x-rays"- (mc/schema- [:map- [:source {:optional true} ::source]- [:root {:optional true} [:ref ::root]]- [:tables {:optional true} any?]- [:query-filter {:optional true} any?]]))+ [:map+ [:source {:optional true} ::source]+ [:root {:optional true} [:ref ::root]]+ [:tables {:optional true} any?]+ [:query-filter {:optional true} [:maybe [:sequential ::filter-clause]]]])(mr/def ::query- "Schema for the type of MBQL queries handled by X-Rays. Currently, either legacy or MBQL 5 is supported, and functions- use [[metabase.xrays.automagic-dashboards.util/do-with-legacy-query]]- or [[metabase.xrays.automagic-dashboards.util/do-with-mbql5-query]] as needed; in the future we can change this to- only accept MBQL 5 and update any code working with legacy MBQL."- [:and+ "Schema for the type of MBQL queries handled by X-Rays."+ [:ref ::lib.schema/query])++(mr/def ::external-op+ [:merge+ ::lib.schema.common/external-op[:map- [:database ::lib.schema.id/database]]- [:multi {:dispatch lib/normalized-mbql-version}- [:mbql-version/mbql5 [:ref ::lib.schema/query]]- [:mbql-version/legacy [:ref ::mbql.s/Query]]]])+ [:args [:sequential [:multi+ {:dispatch coll?, :error/message "Should be a literal or column metadata"}+ [false :any]+ [true ::lib.schema.metadata/column]]]]]])++(mr/def ::aggregation+ [:or ::lib.schema.aggregation/aggregation ::external-op])-(mr/def ::table-id-or-database-id+(mr/def ::metric+ "Schema for an `:xrays/Metric`."[:and[:map- [:table_id {:optional true} [:maybe ::lib.schema.id/table]]+ {:closed true}+ [:name :string]+ [:xrays/aggregation ::aggregation]+ [:table-id {:optional true} [:maybe ::lib.schema.id/table]][:xrays/database-id {:optional true} [:maybe ::lib.schema.id/database]]][:fn- {:error/message "If instance does not have :table_id, it must have :xrays/database-id"}- (some-fn :table_id :xrays/database-id)]])--(mr/def ::field- [:ref ::table-id-or-database-id])+ {:error/message "If instance does not have :table-id, it must have :xrays/database-id"}+ (some-fn :table-id :xrays/database-id)]])-(mr/def ::metric- [:ref ::table-id-or-database-id])--#_(def dashcard- "The base unit thing we are trying to produce in x-rays"- ;; TODO - Beef these specs up, esp. the any?s- (mc/schema- [:map- [:dataset_query {:optional true} [:ref ::query]]- [:dimensions {:optional true} [:sequential string?]]- [:group {:optional true} string?]- [:height pos-int?]- [:metrics {:optional true} any?]- [:position {:optional true} nat-int?]- [:card-score {:optional true} number?]- [:total-score {:optional true} nat-int?]- [:metric-score {:optional true} nat-int?]- [:score-components {:optional true} [:sequential nat-int?]]- [:title {:optional true} string?]- [:visualization {:optional true} any?]- [:width pos-int?]- [:x_label {:optional true} string?]]))--#_(def dashcards- "A bunch of dashcards"- (mc/schema [:maybe [:sequential dashcard]]))--(def field-type+(mr/def ::field-type"A dimension reference, as either a semantic type or entity type and semantic type."- (mc/schema- [:or- [:tuple :keyword]- [:tuple :keyword :keyword]]))+ [:or+ [:tuple :keyword]+ [:tuple :keyword :keyword]])-;;-(def dimension-value+(mr/def ::dimension-value"A specification for the basic keys in the value of a dimension template."- (mc/schema- [:map- [:field_type field-type]- [:score {:optional true} nat-int?]- [:max_cardinality {:optional true} nat-int?]- [:named {:optional true} [:string {:min 1}]]]))+ [:map+ [:field_type ::field-type]+ [:score {:optional true} nat-int?]+ [:max_cardinality {:optional true} nat-int?]+ [:named {:optional true} [:string {:min 1}]]])-(def dimension-template+(mr/def ::dimension-template"A specification for the basic keys in a dimension template."- (mc/schema- [:map-of- {:min 1 :max 1}- [:string {:min 1}]- dimension-value]))+ [:map-of+ {:min 1 :max 1}+ [:string {:min 1}]+ ::dimension-value])-(def metric-value+(mr/def ::metric-value"A specification for the basic keys in the value of a metric template."- (mc/schema- [:map- [:metric [:vector some?]]- [:score {:optional true} nat-int?]- ;[:name some?]- ]))+ [:map+ {:closed true}+ [:metric [:vector some?]]+ [:score {:optional true} nat-int?]+ [:name {:optional true} ::string-or-18n-string]])-(def metric-template+(mr/def ::metric-template"A specification for the basic keys in a metric template."- (mc/schema- [:map-of- {:min 1 :max 1}- [:string {:min 1}]- metric-value]))+ [:map-of+ {:min 1 :max 1}+ [:string {:min 1}]+ ::metric-value])-(def filter-value+(mr/def ::filter-value"A specification for the basic keys in the value of a filter template."- (mc/schema- [:map- [:filter [:vector some?]]- [:score nat-int?]]))+ [:map+ [:filter [:vector some?]]+ [:score nat-int?]])-(def filter-template+(mr/def ::filter-template"A specification for the basic keys in a filter template."- (mc/schema- [:map-of- {:min 1 :max 1}- [:string {:min 1}]- filter-value]))--#_(def card-value- "A specification for the basic keys in the value of a card template."- (mc/schema- [:map- [:dimensions {:optional true} [:vector (mc/schema- [:map-of- {:min 1 :max 1}- [:string {:min 1}]- [:map- [:aggregation {:optional true} string?]]])]]- [:metrics {:optional true} [:vector string?]]- [:filters {:optional true} [:vector string?]]- [:card-score {:optional true} nat-int?]]))--#_(def card-template- "A specification for the basic keys in a card template."- (mc/schema- [:map-of- {:min 1 :max 1}- [:string {:min 1}]- card-value]))--#_(def dashboard-template- "A specification for the basic keys in a dashboard template."- (mc/schema- [:map- [:dimensions {:optional true} [:vector dimension-template]]- [:metrics {:optional true} [:vector metric-template]]- [:filters {:optional true} [:vector filter-template]]- [:cards {:optional true} [:vector card-template]]]))--;; Available values schema -- These are items for which fields have been successfully bound--#_(def available-values- "Specify the shape of things that are available after dimension to field matching for affinity matching"- (mc/schema- [:map- [:available-dimensions [:map-of [:string {:min 1}] any?]]- [:available-metrics [:map-of [:string {:min 1}] any?]]- [:available-filters {:optional true} [:map-of [:string {:min 1}] any?]]]))--;; Schemas for "affinity" functions as these can be particularly confusing--#_(def dimension-set- "A set of dimensions that belong together. This is the basic unity of affinity."- [:set string?])--#_(def semantic-affinity-set- "A set of sematic types that belong together. This is the basic unity of semantic affinity."- [:set :keyword])--#_(def affinity- "A collection of things that go together. In this case, we're a bit specialized on- card affinity, but the key element in the structure is `:base-dims`, which are a- set of dimensions which, when satisfied, enable this affinity object."- (mc/schema- [:map- [:affinity-name :string]- [:affinity-set [:set :keyword]]- [:card-template card-value]- [:metric-constituent-names [:sequential :string]]- [:metric-field-types [:set :keyword]]- [:named-dimensions [:sequential :string]]- [:score {:optional true} nat-int?]]))--#_(def affinities- "A sequence of affinity objects."- (mc/schema- [:sequential affinity]))--#_(def affinity-old- "A collection of things that go together. In this case, we're a bit specialized on- card affinity, but the key element in the structure is `:base-dims`, which are a- set of dimensions which, when satisfied, enable this affinity object."- (mc/schema- [:map- [:dimensions {:optional true} [:vector string?]]- [:metrics {:optional true} [:vector string?]]- [:filters {:optional true} [:vector string?]]- [:score {:optional true} nat-int?]- [:affinity-name string?]- [:base-dims dimension-set]]))--#_(def affinities-old- "A sequence of affinity objects."- (mc/schema- [:sequential affinity-old]))--#_(def affinity-matches- "A map of named affinities to all dimension sets that are associated with this name."- (mc/schema- [:map-of- :string- [:vector dimension-set]]))--(def item+ [:map-of+ {:min 1 :max 1}+ [:string {:min 1}]+ ::filter-value])++(mr/def ::item"A \"thing\" that we bind to, consisting, generally, of at least a name and id"- (mc/schema- [:map- [:id {:optional true} nat-int?]- [:name {:optional true} string?]]))+ [:map+ [:id {:optional true} nat-int?]+ [:name {:optional true} string?]])-(def dim-name->dim-def+(mr/def ::dim-name->dim-def"A map of dimension name to dimension definition."- (mc/schema- [:map-of :string dimension-value]))+ [:map-of :string ::dimension-value])-(def dim-name->matching-fields+(mr/def ::dim-name->matching-fields"A map of named dimensions to a map containing the dimension dataand a sequence of matching items satisfying this dimension"- (mc/schema- [:map-of :string- [:map- [:matches [:sequential item]]]]))+ [:map-of+ :string+ [:map+ [:matches [:sequential ::item]]]])-(def dim-name->dim-defs+matches+(mr/def ::dim-name->dim-defs+matches"The \"full\" grounded dimensions which matches dimension namesto the dimension definition combined with matching fields."- (mut/merge- dim-name->dim-def- dim-name->matching-fields))--#_(def dimension-map- "A map of dimension names to item satisfying that dimensions"- (mc/schema- [:map-of :string item]))--#_(def dimension-maps- "A sequence of dimension maps"- (mc/schema- [:sequential dimension-map]))+ [:merge+ ::dim-name->dim-def+ ::dim-name->matching-fields])-(def normalized-metric-template+(mr/def ::normalized-metric-template"A \"normalized\" metric template is a map containing the metric name as a keyrather than a map of metric name to the map."- (mc/schema- [:map- [:metric-name :string]- [:score nat-int?]- [:metric vector?]]))+ [:map+ {:closed true}+ [:name {:optional true} ::string-or-18n-string]+ [:metric-name :string]+ [:score nat-int?]+ [:metric vector?]])-(def grounded-metric+(mr/def ::grounded-metric.definition+ [:map+ {:closed true}+ [:xrays/aggregations {:optional true} [:maybe [:sequential ::aggregation]]]+ [:xrays/breakouts {:optional true} [:maybe [:sequential ::lib.schema.metadata/column]]]+ [:xrays/filters {:optional true} [:maybe [:sequential ::external-op]]]])++(mr/def ::grounded-metric"A metric containing a definition with actual field references/ids rather than dimension references."- (mc/schema+ [:map+ {:closed true}+ [:metric-name :string]+ [:metric-title :string]+ [:metric-score nat-int?]+ [:metric-definition ::grounded-metric.definition]+ [:id {:optional true} symbol?]+ [:position {:optional true} nat-int?]+ [:dimension-name->field {:optional true} [:map-of :string ::field]]+ [:card-score {:optional true} number?]+ [:score-components {:optional true} [:sequential number?]]+ [:affinity-name {:optional true} :string]+ [:total-score {:optional true} number?]])++(mr/def ::combined-metric+ "A grounded metric in which the metric has been augmented with breakouts."+ [:merge+ ::grounded-metric[:map- [:metric-name :string]- [:metric-title :string]- [:metric-score nat-int?]+ {:closed true}+ [:group {:optional true} :string]+ [:card-name {:optional true} :string]+ [:height {:optional true} number?]+ [:width {:optional true} number?]+ [:title {:optional true} :string]+ [:visualization {:optional true} [:tuple :string :map]]+ [:metrics {:optional true} [:sequential :string]]+ [:filters {:optional true} [:sequential :string]]+ [:description {:optional true} :string]+ [:dimensions {:optional true} [:sequential [:map-of :string :map]]]+ ;; HUH??+ [:order_by {:optional true} [:sequential [:map-of :string [:enum "ascending" "descending"]]]]+ [:limit {:optional true} pos-int?]+ [:x_label {:optional true} :string][:metric-definition- [:map- [:aggregation [:sequential any?]]]]]))--(def combined-metric- "A grounded metric in which the metric has been augmented with breakouts."- (mut/merge- grounded-metric- (mc/schema- [:map- [:metric-definition+ [:merge+ ::grounded-metric.definition[:map- [:aggregation [:sequential any?]]- [:breakout [:sequential any?]]]]])))+ [:xrays/breakouts [:sequential ::lib.schema.metadata/column]]]]]]])-#_(comment- (require '[malli.generator :as mg])- (mg/sample dashboard-template)- (mg/sample affinities)- (mg/sample affinity-matches)- (mg/sample grounded-metric))+(mr/def ::grounded-filter+ [:map+ {:closed true}+ [:score number?]+ [:filter ::external-op]+ [:filter-name :string]])(mr/def ::field[:and@@ -353,3 +272,10 @@out what the schema is supposed to be yet."[:map[:cards {:optional true} [:maybe [:sequential ::card-template]]]])++(mr/def ::grounded-values+ [:map+ {:closed true}+ [:dimensions {:optional true} ::dim-name->matching-fields]+ [:metrics {:optional true} [:sequential ::grounded-metric]]+ [:filters {:optional true} [:sequential ::grounded-filter]]])diff --git a/src/metabase/xrays/automagic_dashboards/util.clj b/src/metabase/xrays/automagic_dashboards/util.cljindex 6b415930ed4..40172b1d800 100644--- a/src/metabase/xrays/automagic_dashboards/util.clj+++ b/src/metabase/xrays/automagic_dashboards/util.clj@@ -4,12 +4,7 @@[clojure.string :as str][medley.core :as m][metabase.analyze.core :as analyze]- ;; legacy usages, do not use legacy MBQL stuff in new code.- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]- ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.legacy-mbql.util :as mbql.u]- [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]- [metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.lib.util.match :as lib.util.match][metabase.models.interface :as mi]@@ -49,32 +44,25 @@[tablespec tables :- [:maybe [:sequential ::ads/source]]](filter #(-> % :entity_type (isa? tablespec)) tables))-(def ^{:arglists '([metric])} saved-metric?+(defn saved-metric?"Is metric a saved (V2) metric? (Note that X-Rays do not currently know how to handle Saved V2 Metrics.)"- ;; legacy usage, do not use legacy MBQL stuff in new code.- #_{:clj-kondo/ignore [:deprecated-var]}- (partial mbql.u/is-clause? :metric))--(def ^{:arglists '([metric])} custom-expression?- "Is this a custom expression?"- ;; legacy usage, do not use legacy MBQL stuff in new code.- #_{:clj-kondo/ignore [:deprecated-var]}- (partial mbql.u/is-clause? :aggregation-options))+ [metric]+ (lib/clause-of-type? metric :metric))(def ^{:arglists '([metric])} adhoc-metric?"Is this an adhoc metric?"- (complement (some-fn saved-metric? custom-expression?)))+ (complement saved-metric?))(def ^{:arglists '([x])} encode-base64-json"Encode given object as form-encoded base-64-encoded JSON."(comp codec/form-encode codec/base64-encode codecs/str->bytes json/encode))-(mu/defn field-reference->id :- [:maybe [:or ms/NonBlankString ms/PositiveInt]]+(mu/defn field-reference->id :- [:maybe [:or ms/NonBlankString ::lib.schema.id/field]]"Extract field ID from a given field reference form."- [clause]- (lib.util.match/match-one clause [:field id _] id))+ [clause :- :mbql.clause/field]+ (lib.util.match/match-one clause [:field _opts id] id))-(mu/defn collect-field-references :- [:maybe [:sequential mbql.s/field]]+(mu/defn collect-field-references :- [:maybe [:sequential :mbql.clause/field]]"Collect all `:field` references from a given form."[form](lib.util.match/match form :field &match))@@ -87,7 +75,8 @@field-id-or-name-or-clause :- [:or::lib.schema.id/fieldms/NonBlankString- ::mbql.s/field-or-expression-ref]]+ :mbql.clause/field+ :mbql.clause/expression]](let [id-or-name (if (sequential? field-id-or-name-or-clause)(field-reference->id field-id-or-name-or-clause)field-id-or-name-or-clause)]@@ -114,28 +103,3 @@"Generate a parameter ID for the given field. In X-ray dashboards a parameter is mapped to a single field only."[field](-> field ((juxt :id :name :unit)) hash str))--(defn do-with-legacy-query- "Call-- (apply f query args)-- with `query` converted to a legacy MBQL query if needed."- [query f & args]- (when (seq query)- (case (lib/normalized-mbql-version query)- :mbql-version/legacy (apply f query args)- :mbql-version/mbql5 (apply f #_{:clj-kondo/ignore [:discouraged-var]} (lib/->legacy-MBQL query) args))))--(defn do-with-mbql5-query- "Call-- (apply f query args)-- with `query` converted to an MBQL 5 query if needed."- [query f & args]- (when (seq query)- (case (lib/normalized-mbql-version query)- :mbql-version/legacy (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]- (apply f (lib-be/normalize-query query) args))- :mbql-version/mbql5 (apply f query args))))diff --git a/src/metabase/xrays/related.clj b/src/metabase/xrays/related.cljindex b408e13ac00..c98b91cb868 100644--- a/src/metabase/xrays/related.clj+++ b/src/metabase/xrays/related.clj@@ -4,12 +4,14 @@[clojure.set :as set][medley.core :as m][metabase.api.common :as api]- ;; legacy usage, do not use legacy MBQL stuff in new code.- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.lib.core :as lib]+ [metabase.lib.schema.util :as lib.schema.util][metabase.models.interface :as mi][metabase.query-processor.util :as qp.util]+ [metabase.segments.schema :as segments.schema]+ [metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- [metabase.xrays.automagic-dashboards.util :as magic.util]+ [metabase.xrays.automagic-dashboards.schema :as ads][toucan2.core :as t2]))(def ^:private ^Long max-best-matches 3)@@ -17,24 +19,22 @@(def ^:private ^Long max-matches (+ max-best-matchesmax-serendipity-matches))-(def ^:private ContextBearingForm+(mr/def ::context-bearing-form[:cat[:and[:or :string :keyword][:fn{:error/message "field, metric, or segment"}(comp #{:field :metric :segment}- qp.util/normalize-token)]]+ keyword)]][:* :any]])(defn- collect-context-bearing-forms[form]- ;; legacy usage, temporary until we convert X-Rays to Lib- (let [form #_{:clj-kondo/ignore [:deprecated-var]} (mbql.normalize/normalize-fragment [:query :filter] form)]- (into #{}- (comp (filter (mr/validator ContextBearingForm))- (map #(update % 0 qp.util/normalize-token)))- (tree-seq sequential? identity form))))+ (into #{}+ (comp (filter (mr/validator ::context-bearing-form))+ (map #(update % 0 qp.util/normalize-token)))+ (tree-seq sequential? identity form)))(defmulti definition"Return the relevant parts of a given entity's definition. Relevant parts are those that carry semantic meaning, and@@ -42,23 +42,23 @@{:arglists '([instance])}mi/model)-(defmethod definition :xrays/Metric- [metric]- (-> metric :definition ((juxt :aggregation :filter))))+(mu/defmethod definition :xrays/Metric+ [metric :- ::ads/metric]+ (-> metric :xrays/aggregation))(defmethod definition :model/Card[card](-> card:dataset_query- (magic.util/do-with-legacy-query #(-> % :query ((juxt :breakout :aggregation :expressions :fields))))))+ ((juxt lib/breakouts lib/aggregations lib/expressions lib/fields))))-(defmethod definition :model/Segment- [segment]- (-> segment :definition :filter))+(mu/defmethod definition :model/Segment+ [segment :- ::segments.schema/segment]+ (-> segment :definition :filter lib/->pMBQL))(defmethod definition :model/Field[field]- [[:field-id (:id field)]])+ [[:field {:lib/uuid (str (random-uuid))} (:id field)]])(defn- similarity"How similar are entities `a` and `b` based on a structural comparison of their@@ -69,8 +69,8 @@(more context-bearing forms), so we just check if the less specifc form is asubset of the more specific one."[a b]- (let [context-a (-> a definition collect-context-bearing-forms)- context-b (-> b definition collect-context-bearing-forms)+ (let [context-a (-> a definition collect-context-bearing-forms lib.schema.util/remove-lib-uuids)+ context-b (-> b definition collect-context-bearing-forms lib.schema.util/remove-lib-uuids)overlap (set/intersection context-a context-b)min-overlap (min (count context-a) (count context-b))](/ (count overlap)diff --git a/src/metabase/xrays/transforms/materialize.clj b/src/metabase/xrays/transforms/materialize.cljindex 171f411e154..73e932b282d 100644--- a/src/metabase/xrays/transforms/materialize.clj+++ b/src/metabase/xrays/transforms/materialize.clj@@ -3,6 +3,7 @@[metabase.api.common :as api][metabase.collections.models.collection :as collection][metabase.lib-be.core :as lib-be]+ [metabase.lib.schema.id :as lib.schema.id][metabase.queries.core :as queries][metabase.queries.schema :as queries.schema][metabase.query-processor.preprocess :as qp.preprocess]@@ -18,12 +19,13 @@(t2/select-one [:model/Collection :location :id]:id (get-or-create-root-container-collection!))))-(defn get-collection+(mu/defn get-collection :- [:maybe ::lib.schema.id/collection]"Get collection named `collection-name`. If no location is given root collection for automaticallygenerated transforms is assumed (see `get-or-create-root-container-collection!`)."([collection-name](get-collection collection-name (root-container-location)))- ([collection-name location]+ ([collection-name :- :string+ location :- :string](t2/select-one-pk :model/Collection:name collection-name:location location)))diff --git a/test/metabase/parameters/schema_test.cljc b/test/metabase/parameters/schema_test.cljcindex 27c0e7ffe5c..5fa8c3f48a6 100644--- a/test/metabase/parameters/schema_test.cljc+++ b/test/metabase/parameters/schema_test.cljc@@ -18,6 +18,6 @@(parameters.schema/normalize-parameter-mapping{"parameter_id" "53bb6214", "card_id" -1, "target" ["variable" ["template-tag" "id"]]})))))-(deftest ^:paralell normalize-parameters-without-adding-default-type-test+(deftest ^:parallel normalize-parameters-without-adding-default-type-test(is (= [{:id "x", :target [:dimension [:template-tag "y"]]}](parameters.schema/normalize-parameters-without-adding-default-types [{"id" "x", "target" ["dimension" ["template-tag" "y"]]}]))))diff --git a/test/metabase/xrays/api/automagic_dashboards_test.clj b/test/metabase/xrays/api/automagic_dashboards_test.cljindex 0904983667e..8a89ae8f19b 100644--- a/test/metabase/xrays/api/automagic_dashboards_test.clj+++ b/test/metabase/xrays/api/automagic_dashboards_test.clj@@ -238,12 +238,10 @@(is (some?(api-call! "table/%s/compare/segment/%s"[(mt/id :venues) segment-id]))))-(testing "GET /api/automagic-dashboards/table/:id/rule/example/indepth/compare/segment/:segment-id"(is (some?(api-call! "table/%s/rule/example/indepth/compare/segment/%s"[(mt/id :venues) segment-id]))))-(testing "GET /api/automagic-dashboards/adhoc/:id/cell/:cell-query/compare/segment/:segment-id"(is (some?(api-call! "adhoc/%s/cell/%s/compare/segment/%s"@@ -611,7 +609,7 @@{:id (mt/id :categories)}])}(mt/user-http-request :crowberto :get 200 (str "automagic-dashboards/table/" (mt/id :venues) "/query_metadata")))))-(deftest ^:paralell allow-equals-in-base-64-encoding-for-now-test+(deftest ^:parallel allow-equals-in-base-64-encoding-for-now-test(testing "Allow `=` in the route part of X-Rays API requests... for now"(doseq [schema [::api.magic/base-64-encoded-json::api.magic/entity-id-or-query]]diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.cljindex 811fdd32bda..5cf71b585a8 100644--- a/test/metabase/xrays/automagic_dashboards/core_test.clj+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj@@ -9,6 +9,7 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.test-metadata :as meta][metabase.lib.util.match :as lib.util.match][metabase.models.interface :as mi][metabase.permissions.models.permissions :as perms]@@ -26,15 +27,18 @@[metabase.xrays.automagic-dashboards.core :as magic][metabase.xrays.automagic-dashboards.dashboard-templates :as dashboard-templates][metabase.xrays.automagic-dashboards.interesting :as interesting]- [metabase.xrays.automagic-dashboards.util :as magic.util][metabase.xrays.test-util.automagic-dashboards :as automagic-dashboards.test][toucan2.core :as t2]))(set! *warn-on-reflection* true)+(deftest ^:parallel normalize-query-filter-test+ (is (=? {:query-filter [[:= {} [:field {} 1] 2]]}+ (lib/normalize ::magic/automagic-analysis.opts {:query-filter [:= [:field 1 nil] 2]}))))+;;; ------------------- Dashboard template matching --------------------(deftest dashboard-template-matching-test+(deftest ^:parallel dashboard-template-matching-test(is (= [:entity/UserTable :entity/GenericTable :entity/*](->> (mt/id :users)(t2/select-one :model/Table :id)@@ -42,7 +46,7 @@(#'magic/matching-dashboard-templates (dashboard-templates/get-dashboard-templates ["table"]))(map (comp first :applies_to))))))-(deftest dashboard-template-matching-test-2+(deftest ^:parallel dashboard-template-matching-test-2(testing "Test fallback to GenericTable"(is (= [:entity/GenericTable :entity/*](->> (-> (t2/select-one :model/Table :id (mt/id :users))@@ -53,7 +57,7 @@;;; ------------------- `->root source` --------------------(deftest source-root-table-test+(deftest ^:parallel source-root-table-test(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Table"(mt/dataset test-data(testing "The source of a table is the table itself"@@ -63,7 +67,7 @@(is (= entity table))(is (= source entity)))))))-(deftest source-root-field-test+(deftest ^:parallel source-root-field-test(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Field"(mt/dataset test-data(testing "The source of a field is the originating table of the field"@@ -73,7 +77,7 @@(is (= source table))(is (= entity field)))))))-(deftest source-root-card-test+(deftest ^:parallel source-root-card-test(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Card"(mt/dataset test-data(testing "Card sourcing has four branches..."@@ -92,7 +96,7 @@(is (=? (dissoc source :dataset_query)(assoc card :entity_type :entity/GenericTable))))))))))-(deftest source-root-card-test-2+(deftest ^:parallel source-root-card-test-2(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Card"(mt/dataset test-data(testing "Card sourcing has four branches..."@@ -118,7 +122,7 @@(is (=? (dissoc source :entity_type)nested-query)))))))))-(deftest source-root-card-test-3+(deftest ^:parallel source-root-card-test-3(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Card"(mt/dataset test-data(testing "Card sourcing has four branches..."@@ -134,7 +138,7 @@(is (=? (dissoc source :dataset_query :entity_type)card))))))))))-(deftest source-root-card-test-4+(deftest ^:parallel source-root-card-test-4(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Card"(mt/dataset test-data(testing "Card sourcing has four branches..."@@ -155,7 +159,7 @@card))(is (= source (t2/select-one :model/Table :id table-id))))))))))-(deftest source-root-query-test+(deftest ^:parallel source-root-query-test(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Query"(mt/dataset test-data(testing "The source of a query is the underlying datasource of the query"@@ -163,15 +167,17 @@:model/Query{:database-id (mt/id):table-id (mt/id :orders)- :dataset_query {:database (mt/id)- :type :query- :query {:source-table (mt/id :orders)- :aggregation [[:count]]}}})+ :dataset_query (lib/query+ (mt/metadata-provider)+ {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :orders)+ :aggregation [[:count]]}})}){:keys [entity source]} (#'magic/->root query)](is (= entity query))(is (= source (t2/select-one :model/Table (mt/id :orders)))))))))-(deftest source-root-segment-test+(deftest ^:parallel source-root-segment-test(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Segment"(testing "The source of a segment is its underlying table."(mt/with-temp [:model/Segment segment {:table_id (mt/id :venues)@@ -198,7 +204,7 @@;; These test names were named by staring at them for a while, so they may be misleading-(deftest automagic-analysis-test+(deftest ^:parallel automagic-analysis-test(mt/with-test-user :rasta(automagic-dashboards.test/with-rollback-only-transaction(doseq [[table cardinality] (map vector@@ -206,7 +212,7 @@[2 8 11 11 15 17 5 7])](test-automagic-analysis table cardinality)))))-(deftest automagic-analysis-test-2+(deftest ^:parallel automagic-analysis-test-2(mt/with-test-user :rasta(automagic-dashboards.test/with-rollback-only-transaction(is (= 1@@ -215,7 +221,7 @@(filter :card)count))))))-(deftest weird-characters-in-names-test+(deftest ^:parallel weird-characters-in-names-test(mt/with-test-user :rasta(automagic-dashboards.test/with-rollback-only-transaction(-> (t2/select-one :model/Table :id (mt/id :venues))@@ -224,7 +230,7 @@;; Cardinality of cards genned from fields is much more labile than anything else;; Not just with respect to drivers, but all sorts of other stuff that makes it chaotic-(deftest mass-field-test+(deftest ^:parallel mass-field-test(mt/with-test-user :rasta(automagic-dashboards.test/with-rollback-only-transaction(doseq [field (t2/select :model/Field@@ -233,7 +239,7 @@{:order-by [[:id :asc]]})](is (pos? (count (:dashcards (magic/automagic-analysis field {})))))))))-(deftest parameter-mapping-test+(deftest ^:parallel parameter-mapping-test(mt/dataset test-data(testing "mbql queries have parameter mappings with field ids"(let [table (t2/select-one :model/Table :id (mt/id :products))@@ -246,7 +252,7 @@(:dashcards dashboard))](is (= expected-targets actual-targets))))))-(deftest parameter-mapping-test-2+(deftest ^:parallel parameter-mapping-test-2(mt/dataset test-data(testing "native queries have parameter mappings with field ids"(let [query (mt/native-query {:query "select * from products"})]@@ -607,7 +613,7 @@(for [dashcard (:dashcards dashboard):let [query (get-in dashcard [:card :dataset_query])]:when query- :let [breakouts (magic.util/do-with-mbql5-query query lib/breakouts)]+ :let [breakouts (lib/breakouts query)]id (lib.util.match/match breakouts[:field (_opts :guard :binning) (id :guard pos-int?)]id)]@@ -641,7 +647,7 @@temporal-field-ids (for [dashcard (:dashcards dashboard):let [query (get-in dashcard [:card :dataset_query])]:when query- :let [breakouts (magic.util/do-with-mbql5-query query lib/breakouts)]+ :let [breakouts (lib/breakouts query)]id (lib.util.match/match breakouts[:field (_opts :guard :temporal-unit) (id :guard pos-int?)]id)]@@ -756,7 +762,7 @@(is (false? (str/ends-with? question-dashboard-name "model")))(is (true? (str/ends-with? question-dashboard-name (format "\"%s\"" (:name question-card))))))))))))-(deftest model-based-automagic-dashboards-have-correct-parameter-mappings-test+(deftest ^:parallel model-based-automagic-dashboards-have-correct-parameter-mappings-test(testing "Dashcard parameter mappings have valid targets when X-raying models (#58214)"(mt/dataset test-data(mt/with-temp@@ -907,14 +913,29 @@(mt/with-test-user :rasta(automagic-dashboards.test/with-rollback-only-transaction(perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id) =- (let [all-dashcard-filters (->> (magic/automagic-analysis entity {:cell-query cell-query :show :all})- :dashcards- (keep #(-> %- (get-in [:card :dataset_query])- (magic.util/do-with-legacy-query get-in [:query :filter]))))- filter-contains-cell-query? #(= cell-query (some #{cell-query} %))]- (is (pos? (count all-dashcard-filters)))- (is (every? filter-contains-cell-query? all-dashcard-filters))))))))))+ (let [dashcards (:dashcards (magic/automagic-analysis entity {:cell-query cell-query :show :all}))]+ (is (= ["# Summary"+ "Total Venues"+ "Distinct Category ID"+ "# Where these Venues are"+ "Venues by coordinates"+ "# How these Venues are distributed"+ "Venues per Price"]+ (map (some-fn (comp :name :card) (comp :text :visualization_settings)) dashcards)))+ (let [all-dashcard-filters (keep #(some-> %+ (get-in [:card :dataset_query])+ not-empty+ lib/filters)+ dashcards)]+ (is (=? [[[:> {} [:field {:base-type :type/Integer} (mt/id :venues :price)] 10]+ [:= {} [:field {} (mt/id :venues :category_id)] 2]]+ [[:> {} [:field {:base-type :type/Integer} (mt/id :venues :price)] 10]+ [:= {} [:field {} (mt/id :venues :category_id)] 2]]+ [[:> {} [:field {:base-type :type/Integer} (mt/id :venues :price)] 10]+ [:= {} [:field {} (mt/id :venues :category_id)] 2]]+ [[:> {} [:field {:base-type :type/Integer} (mt/id :venues :price)] 10]+ [:= {} [:field {} (mt/id :venues :category_id)] 2]]]+ all-dashcard-filters)))))))))))(deftest complicated-card-cell-test(mt/with-non-admin-groups-no-root-collection-perms@@ -931,7 +952,7 @@(-> (t2/select-one :model/Card :id card-id)(test-automagic-analysis [:= [:field (mt/id :venues :category_id) nil] 2] 7)))))))-(deftest adhoc-filter-test+(deftest ^:parallel adhoc-filter-test(mt/with-test-user :rasta(automagic-dashboards.test/with-rollback-only-transaction(let [q (api.automagic-dashboards/adhoc-query-instance {:query {:filter [:> [:field (mt/id :venues :price) nil] 10]@@ -940,7 +961,7 @@:database (mt/id)})](test-automagic-analysis q 7)))))-(deftest adhoc-count-test+(deftest ^:parallel adhoc-count-test(mt/with-test-user :rasta(automagic-dashboards.test/with-rollback-only-transaction(let [q (api.automagic-dashboards/adhoc-query-instance {:query {:aggregation [[:count]]@@ -950,7 +971,7 @@:database (mt/id)})](test-automagic-analysis q 17)))))-(deftest adhoc-fk-breakout-test+(deftest ^:parallel adhoc-fk-breakout-test(mt/with-test-user :rasta(automagic-dashboards.test/with-rollback-only-transaction(let [q (api.automagic-dashboards/adhoc-query-instance {:query {:aggregation [[:count]]@@ -960,7 +981,7 @@:database (mt/id)})](test-automagic-analysis q 9)))))-(deftest adhoc-filter-cell-test+(deftest ^:parallel adhoc-filter-cell-test(mt/with-test-user :rasta(automagic-dashboards.test/with-rollback-only-transaction(let [q (api.automagic-dashboards/adhoc-query-instance {:query {:filter [:> [:field (mt/id :venues :price) nil] 10]@@ -969,22 +990,27 @@:database (mt/id)})](test-automagic-analysis q [:= [:field (mt/id :venues :category_id) nil] 2] 7)))))-(deftest join-splicing-test+(deftest ^:parallel join-splicing-test(mt/with-test-user :rasta(automagic-dashboards.test/with-rollback-only-transaction(let [join-vec [{:source-table (mt/id :categories):condition [:= [:field (mt/id :categories :id) nil] 1]:strategy :left-join:alias "Dealios"}]- q (api.automagic-dashboards/adhoc-query-instance {:query {:source-table (mt/id :venues)- :joins join-vec- :aggregation [[:sum [:field (mt/id :categories :id) {:join-alias "Dealios"}]]]}- :type :query+ q (api.automagic-dashboards/adhoc-query-instance {:query {:source-table (mt/id :venues)+ :joins join-vec+ :aggregation [[:sum [:field (mt/id :categories :id) {:join-alias "Dealios"}]]]}+ :type :query:database (mt/id)})res (magic/automagic-analysis q {})cards (vec (:dashcards res))- join-member (get-in cards [2 :card :dataset_query :query :joins])]- (is (= join-vec join-member))))))+ join-member (lib/joins (get-in cards [2 :card :dataset_query]))]+ (is (=? [{:strategy :left-join+ :alias "Dealios"+ :conditions [[:= {} [:field {} (mt/id :categories :id)] 1]]+ :lib/type :mbql/join+ :stages [{:lib/type :mbql.stage/mbql, :source-table (mt/id :categories)}]}]+ join-member))))));;; ------------------- /candidates -------------------@@ -999,7 +1025,7 @@[:* :any]](magic/candidate-tables (mt/db)))))))))-(deftest candidates-test-2+(deftest ^:parallel candidates-test-2(testing "/candidates"(testing "should work with unanalyzed tables"(mt/with-test-user :rasta@@ -1010,7 +1036,7 @@(is (=? [{:tables [{:table {:id table-id}}]}](magic/candidate-tables (t2/select-one :model/Database :id db-id)))))))))-(deftest call-count-test+(deftest ^:parallel call-count-test(mt/with-temp [:model/Database {db-id :id} {}:model/Table {table-id :id} {:db_id db-id}:model/Field _ {:table_id table-id}@@ -1022,7 +1048,7 @@(magic/candidate-tables database)(is (= 2 (call-count)))))))))-(deftest empty-table-test+(deftest ^:parallel empty-table-test(testing "candidate-tables should work with an empty Table (no Fields)"(mt/with-temp [:model/Database db {}:model/Table _ {:db_id (:id db)}]@@ -1030,7 +1056,7 @@(is (= [](magic/candidate-tables db)))))))-(deftest enhance-table-stats-test+(deftest ^:parallel enhance-table-stats-test(mt/with-temp [:model/Database {db-id :id} {}:model/Table {table-id :id} {:db_id db-id}:model/Field _ {:table_id table-id :semantic_type :type/PK}@@ -1042,7 +1068,7 @@(-> (#'magic/load-tables-with-enhanced-table-stats [[:= :id table-id]])first)))))))-(deftest enhance-table-stats-fk-test+(deftest ^:parallel enhance-table-stats-fk-test(mt/with-temp [:model/Database {db-id :id} {}:model/Table {table-id :id} {:db_id db-id}:model/Field _ {:table_id table-id :semantic_type :type/PK}@@ -1056,7 +1082,7 @@;;; ------------------- Definition overloading --------------------(deftest most-specific-definition-test+(deftest ^:parallel most-specific-definition-test(testing "Identity"(is (= :d1(-> [{:d1 {:field_type [:type/Category] :score 100}}]@@ -1064,7 +1090,7 @@firstkey)))))-(deftest ancestors-definition-test+(deftest ^:parallel ancestors-definition-test(testing "Base case: more ancestors"(is (= :d2(-> [{:d1 {:field_type [:type/Category] :score 100}}@@ -1073,7 +1099,7 @@firstkey)))))-(deftest definition-tiebreak-test+(deftest ^:parallel definition-tiebreak-test(testing "Break ties based on the number of additional filters"(is (= :d3(-> [{:d1 {:field_type [:type/Category] :score 100}}@@ -1085,7 +1111,7 @@firstkey)))))-(deftest definition-tiebreak-score-test+(deftest ^:parallel definition-tiebreak-score-test(testing "Break ties on score"(is (= :d2(-> [{:d1 {:field_type [:type/Category] :score 100}}@@ -1095,7 +1121,7 @@firstkey)))))-(deftest definition-tiebreak-precedence-test+(deftest ^:parallel definition-tiebreak-precedence-test(testing "Number of additional filters has precedence over score"(is (= :d3(-> [{:d1 {:field_type [:type/Category] :score 100}}@@ -1109,7 +1135,7 @@;;; -------------------- Filters ---------------------(deftest most-specific-definition-inner-shape-test+(deftest ^:parallel most-specific-definition-inner-shape-test(testing "Ensure we have examples to understand the shape returned from most-specific-definition"(mt/dataset test-data(testing ""@@ -1179,7 +1205,7 @@candidate-bindings(#'interesting/most-specific-matched-dimension))))))))))))-(deftest bind-dimensions-inner-shape-test+(deftest ^:parallel bind-dimensions-inner-shape-test(testing "Ensure we have examples to understand the shape returned from bind-dimensions"(mt/dataset test-data(testing "Clearly demonstrate the mechanism of full dimension binding"@@ -1212,7 +1238,7 @@:score 60}}(update-in bound-dimensions ["Loc" :matches] (partial sort-by :id))))))))))-(deftest binding-functions-with-all-same-names-and-types-test+(deftest ^:parallel binding-functions-with-all-same-names-and-types-test(testing "Ensure expected behavior when multiple columns alias to the same base column and display metadata uses thesame name for all columns."(mt/dataset test-data@@ -1279,7 +1305,7 @@;;; -------------------- Ensure generation of subcards via related (includes indepth, drilldown) ---------------------(deftest related-card-generation-test+(deftest ^:parallel related-card-generation-test(testing "Ensure that the `related` function is called and the right cards are created."(mt/with-test-user :rasta(mt/dataset test-data@@ -1313,17 +1339,18 @@(update :zoom-in (comp vec (partial sort-by :title)))(update :related (comp vec (partial sort-by :title)))))))))))-(deftest singular-cell-dimensions-test+(deftest ^:parallel singular-cell-dimensions-test(testing "Find the cell dimensions for a cell query"(is (= #{1 2 "TOTAL"}(#'magic/singular-cell-dimension-field-ids- {:cell-query- [:and- [:= [:field 1 nil]]- [:= [:field 2 nil]]- [:= [:field "TOTAL" {:base-type :type/Number}]]]})))))--(deftest adhoc-query-with-explicit-joins-14793-test+ {:database 1+ :cell-query (lib/normalize+ [:and+ [:= [:field 1 nil] 5]+ [:= [:field 2 nil] 6]+ [:= [:field "TOTAL" {:base-type :type/Number}] 7]])})))))++(deftest ^:parallel adhoc-query-with-explicit-joins-14793-test(testing "A verification of the fix for https://github.com/metabase/metabase/issues/14793,X-rays fails on explicit joins, when metric is for the joined table"(mt/dataset test-data@@ -1360,7 +1387,7 @@(testing "An expectation check -- this dashboard should produce this group heading"(is (contains? section-headings expected-section)))))))))-(deftest compare-to-the-rest-25278+32557-test+(deftest ^:parallel compare-to-the-rest-25278+32557-test(testing "Ensure valid queries are generated for an automatic comparison dashboard (fixes 25278 & 32557)"(mt/dataset test-data(mt/with-test-user :crowberto@@ -1390,12 +1417,16 @@(is (some? card-with-cell-query))(is (some? card-without-cell-query)))(testing "The cell-query exists in only one of the cards"- (is (not= cell-query (get-in base-query [:query :filter])))- (is (= cell-query (get-in filtered-query [:query :filter]))))+ (is (nil? (get-in base-query [:query :filter])))+ (is (=? [[:=+ {}+ [:field {:base-type :type/Text, :join-alias "Products"} (mt/id :products :title)]+ "Intelligent Granite Hat"]]+ (lib/filters filtered-query))))(testing "Join aliases exist in the queries";; The reason issues 25278 & 32557 would blow up is the lack of join aliases.- (is (= ["Products"] (map :alias (get-in base-query [:query :joins]))))- (is (= ["Products"] (map :alias (get-in filtered-query [:query :joins])))))+ (is (= ["Products"] (map :alias (lib/joins base-query))))+ (is (= ["Products"] (map :alias (lib/joins filtered-query)))))(testing "Card queries are both executable and produce different results";; Note that issues 25278 & 32557 would blow up on the filtered queries.(let [base-data (get-in (qp/process-query base-query) [:data :rows])@@ -1404,7 +1435,7 @@(is (some? filtered-data))(is (not= base-data filtered-data)))))))))-(deftest compare-to-the-rest-with-expression-16680-test+(deftest ^:parallel compare-to-the-rest-with-expression-16680-test(testing "Ensure a valid comparison dashboard is generated with custom expressions (fixes 16680)"(mt/dataset test-data(mt/with-test-user :crowberto@@ -1437,7 +1468,7 @@series-card-label "Number of Orders per day of the week (Number of Orders where TestColumn is 2 and Created At is in February 2019)"{[{series-dataset-query :dataset_query}] :series- {card-dataset-query :dataset_query} :card+ {card-dataset-query :dataset_query} :card:as series-card} (some(fn [{{card-name :name} :card :as dashcard}](when (= series-card-label card-name)@@ -1448,11 +1479,16 @@(is (some? card-with-cell-query))(is (some? card-without-cell-query)))(testing "The cell-query exists in only one of the cards"- (is (not= cell-query (get-in base-query [:query :filter])))- (is (= cell-query (get-in filtered-query [:query :filter]))))+ (is (empty? (lib/filters base-query)))+ (is (=? [[:and {}+ [:= {} [:expression {} "TestColumn"] 2]+ [:= {}+ [:field {:temporal-unit :month} (mt/id :orders :created_at)]+ "2019-02-01T00:00:00Z"]]]+ (lib/filters filtered-query))))(testing "Expressions exist in the queries"- (is (= {"TestColumn" [:+ 1 1]} (get-in base-query [:query :expressions])))- (is (= {"TestColumn" [:+ 1 1]} (get-in filtered-query [:query :expressions]))))+ (is (=? [[:+ {:lib/expression-name "TestColumn"} 1 1]] (lib/expressions base-query)))+ (is (=? [[:+ {:lib/expression-name "TestColumn"} 1 1]] (lib/expressions filtered-query))))(testing "Card queries are both executable and produce different results"(let [base-data (get-in (qp/process-query base-query) [:data :rows])filtered-data (get-in (qp/process-query filtered-query) [:data :rows])]@@ -1470,42 +1506,48 @@(is (some? card-dataset-query))(is (= 7 (:row_count (qp/process-query card-dataset-query)))))))))))-(deftest preserve-entity-element-test+(deftest ^:parallel preserve-entity-element-test(testing "Join preservation scenarios: merge, empty expressions, no expressions, no card"- (is (= [[{:strategy :left-join, :alias "Orders"}- {:strategy :left-join, :alias "Products"}]- [{:strategy :left-join, :alias "Products"}]- [{:strategy :left-join, :alias "Products"}]+ (is (= [[{:alias "Orders"}+ {:alias "Products"}]+ [{:alias "Products"}]+ [{:alias "Products"}]nil](->>- (#'magic/preserve-entity-element- {:dashcards [{:card {:dataset_query {:database 1, :type :query, :query {:joins [{:strategy :left-join :alias "Orders"}]}}}}- {:card {:dataset_query {:database 1, :type :query, :query {:joins []}}}}- {:card {:dataset_query {:database 1, :type :query, :query {}}}}+ (#'magic/preserve-joins+ {:dashcards [{:card {:dataset_query (-> (lib/query meta/metadata-provider (meta/table-metadata :reviews))+ (lib/join (-> (lib/join-clause (meta/table-metadata :orders))+ (lib/with-join-conditions [(lib/= (meta/field-metadata :reviews :id)+ (meta/field-metadata :orders :id))]))))}}+ {:card {:dataset_query (lib/query meta/metadata-provider (meta/table-metadata :reviews))}}+ {:card {:dataset_query (lib/query meta/metadata-provider (meta/table-metadata :reviews))}}{:viz_settings nil}]}- {:dataset_query {:database 1, :type :query, :query {:joins [{:strategy :left-join :alias "Products"}]}}}- :joins)+ {:dataset_query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/join (meta/table-metadata :products)))}):dashcards- (mapv (comp :joins :query :dataset_query :card)))))))+ (mapv (fn [dashcard]+ (some-> dashcard :card :dataset_query not-empty lib/joins (->> (map #(select-keys % [:strategy :alias])))))))))))-(deftest preserve-entity-element-test-2+(deftest ^:parallel preserve-entity-element-test-2(testing "Expression preservation scenarios: merge, empty expressions, no expressions, no card"- (is (= [{"Existing" [:- 1 1] "TestColumn" [:+ 1 1]}- {"TestColumn" [:+ 1 1]}- {"TestColumn" [:+ 1 1]}- nil]- (->>- (#'magic/preserve-entity-element- {:dashcards [{:card {:dataset_query {:database 1, :type :query, :query {:expressions {"Existing" [:- 1 1]}}}}}- {:card {:dataset_query {:database 1, :type :query, :query {:expressions {}}}}}- {:card {:dataset_query {:database 1, :type :query, :query {}}}}- {:viz_settings nil}]}- {:dataset_query {:database 1, :type :query, :query {:expressions {"TestColumn" [:+ 1 1]}}}}- :expressions)- :dashcards- (mapv (comp :expressions :query :dataset_query :card)))))))--(deftest compare-to-the-rest-15655-test+ (is (=? [[[:- {:lib/expression-name "Existing"} 1 1]+ [:+ {:lib/expression-name "TestColumn"} 1 1]]+ [[:+ {:lib/expression-name "TestColumn"} 1 1]]+ [[:+ {:lib/expression-name "TestColumn"} 1 1]]+ nil]+ (->>+ (#'magic/preserve-expressions+ {:dashcards [{:card {:dataset_query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/expression "Existing" (lib/- 1 1)))}}+ {:card {:dataset_query (lib/query meta/metadata-provider (meta/table-metadata :venues))}}+ {:card {:dataset_query (lib/query meta/metadata-provider (meta/table-metadata :venues))}}+ {:viz_settings nil}]}+ {:dataset_query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/expression "TestColumn" (lib/+ 1 1)))})+ :dashcards+ (mapv #(some-> % :card :dataset_query not-empty lib/expressions)))))))++(deftest ^:parallel compare-to-the-rest-15655-test(testing "Questions based on native questions should produce a valid dashboard. (#15655)"(let [native-query {:native {:query "select * from people limit 1"}:type :native@@ -1570,7 +1612,7 @@;;; this is a fixed version of the test above that correctly generates the second card so that it matches the equivalent;;; Cypress test spec.-(deftest compare-to-the-rest-15655-fixed-test+(deftest ^:parallel compare-to-the-rest-15655-fixed-test(testing "Questions based on native questions should produce a valid dashboard. (#15655)"(mt/with-temp [:model/Card {native-card-id :id} (merge (mt/card-with-source-metadata-for-query(lib/native-query (mt/metadata-provider) "select * from people LIMIT 100;"))@@ -1607,7 +1649,7 @@;; and "cell-query" base-64 encoded(:dashcards (magic/automagic-analysis card {:cell-query cell-query})))))))))-(deftest source-fields-are-populated-for-aggregations-38618-test+(deftest ^:parallel source-fields-are-populated-for-aggregations-38618-test(testing "X-ray aggregates (metrics) with source fields in external tables should properly fill in `:source-field` (#38618)"(mt/dataset test-data(let [dashcard (->> (magic/automagic-analysis (t2/select-one :model/Table :id (mt/id :reviews)) {:show :all})@@ -1618,9 +1660,11 @@first)aggregate (-> dashcard(get-in [:card :dataset_query])- (magic.util/do-with-legacy-query get-in [:query :aggregation]))]+ lib/aggregations)](testing "Fields requiring a join should have :source-field populated in the aggregate."- (is (= [[:distinct [:field (mt/id :products :id)- ;; This should be present vs. nil (value before issue)- {:source-field (mt/id :reviews :product_id)}]]]- aggregate)))))))+ (is (=? [[:distinct {}+ [:field+ ;; This should be present vs. nil (value before issue)+ {:source-field (mt/id :reviews :product_id)}+ (mt/id :products :id)]]]+ aggregate)))))))diff --git a/test/metabase/xrays/automagic_dashboards/filters_test.clj b/test/metabase/xrays/automagic_dashboards/filters_test.cljindex 420bd405396..177606f7750 100644--- a/test/metabase/xrays/automagic_dashboards/filters_test.clj+++ b/test/metabase/xrays/automagic_dashboards/filters_test.clj@@ -1,35 +1,37 @@(ns metabase.xrays.automagic-dashboards.filters-test(:require[clojure.test :refer :all]+ [metabase.lib.core :as lib][metabase.xrays.automagic-dashboards.filters :as filters]))(deftest ^:parallel replace-date-range-test(testing "Replace range with the more specific `:=`."- (is (= [:and- [:= [:field 2 nil] 42]- [:= [:field 9 {:source-field 1}] "foo"]]- (filters/inject-refinement- [:and- [:= [:field 9 {:source-field 1}] "foo"]- [:and- [:> [:field 2 nil] 10]- [:< [:field 2 nil] 100]]]- [:= [:field 2 nil] 42])))))+ (is (=? [[:= {} [:field {:source-field 1} 9] "foo"]+ [:= {} [:field {} 2] 42]]+ (filters/inject-refinement+ (mapv+ lib/normalize+ [[:= [:field 9 {:source-field 1}] "foo"]+ [:and+ [:> [:field 2 nil] 10]+ [:< [:field 2 nil] 100]]])+ (lib/normalize+ [:= [:field 2 nil] 42]))))))(deftest ^:parallel merge-using-and-test(testing "If there's no overlap between filter clauses, just merge using `:and`."- (is (= [:and- [:= [:field 3 nil] 42]- [:= [:field 9 {:source-field 1}] "foo"]- [:> [:field 2 nil] 10]- [:< [:field 2 nil] 100]]- (filters/inject-refinement- [:and- [:= [:field 9 {:source-field 1}] "foo"]- [:and- [:> [:field 2 nil] 10]- [:< [:field 2 nil] 100]]]- [:= [:field 3 nil] 42])))))+ (is (=? [[:= {} [:field {:source-field 1} 9] "foo"]+ [:> {} [:field {} 2] 10]+ [:< {} [:field {} 2] 100]+ [:= {} [:field {} 3] 42]]+ (filters/inject-refinement+ (mapv+ lib/normalize+ [[:= [:field 9 {:source-field 1}] "foo"]+ [:and+ [:> [:field 2 nil] 10]+ [:< [:field 2 nil] 100]]])+ (lib/normalize [:= [:field 3 nil] 42]))))))(deftest ^:parallel interesting-fields-test(testing "Should return only :type/Temporal and :type/Boolean fields, or fields with the :type/Category semantic type"diff --git a/test/metabase/xrays/automagic_dashboards/interesting_test.clj b/test/metabase/xrays/automagic_dashboards/interesting_test.cljindex afcb0675e3a..b5e63830cd9 100644--- a/test/metabase/xrays/automagic_dashboards/interesting_test.clj+++ b/test/metabase/xrays/automagic_dashboards/interesting_test.clj@@ -1,7 +1,6 @@(ns metabase.xrays.automagic-dashboards.interesting-test(:require[clojure.test :refer :all]- [metabase.models.interface :as mi][metabase.query-processor.metadata :as qp.metadata][metabase.test :as mt][metabase.xrays.automagic-dashboards.core :as magic]@@ -9,20 +8,6 @@[metabase.xrays.automagic-dashboards.util :as magic.util][toucan2.core :as t2]))-;;; ------------------- `->reference` ---------------------(deftest ^:parallel ->reference-test- (is (= [:field 1 nil]- (->> (assoc (mi/instance :model/Field) :id 1)- (#'interesting/->reference :mbql))))-- (is (= [:field 2 {:source-field 1}]- (->> (assoc (mi/instance :model/Field) :id 1 :fk_target_field_id 2)- (#'interesting/->reference :mbql))))- (is (= 42- (->> 42- (#'interesting/->reference :mbql)))))-;;; -------------------- Bind dimensions, candidate bindings, field candidates, and related --------------------(defn field [table column]@@ -267,7 +252,7 @@:matches [{:name "Number of Greebles"}]}}]b-lt-a-bindings [{"Quantity" {:score 100:matches [{:name "Number of Nurnies"}]}}- {"Quantity" {:score 1,+ {"Quantity" {:score 1:matches [{:name "Number of Greebles"}]}}]bind-dimensions-merge-fn #(apply merge-with (fn [a b](case (compare (:score a) (:score b))@@ -307,57 +292,57 @@quantity-dimunmatched-dim]bindings (#'interesting/find-dimensions context dimensions)]- (is (= {"Quantity" {:field_type [:entity/GenericTable :type/Quantity],- :score 100,- :matches [{:base_type :type/Integer,- :name "Number of Nurnies",- :semantic_type :type/Quantity,- :link nil,- :field_type [:entity/GenericTable :type/Quantity],+ (is (= {"Quantity" {:field_type [:entity/GenericTable :type/Quantity]+ :score 100+ :matches [{:base_type :type/Integer+ :name "Number of Nurnies"+ :semantic_type :type/Quantity+ :link nil+ :field_type [:entity/GenericTable :type/Quantity]:score 100}- {:base_type :type/Integer,- :name "Number of Greebles",- :semantic_type :type/Quantity,- :link nil,- :field_type [:entity/GenericTable :type/Quantity],- :score 100}]},- "GenericNumber" {:field_type [:entity/GenericTable :type/Number],- :score 80,- :matches [{:base_type :type/Float,- :name "A double number field",- :link nil,- :field_type [:entity/GenericTable :type/Number],+ {:base_type :type/Integer+ :name "Number of Greebles"+ :semantic_type :type/Quantity+ :link nil+ :field_type [:entity/GenericTable :type/Quantity]+ :score 100}]}+ "GenericNumber" {:field_type [:entity/GenericTable :type/Number]+ :score 80+ :matches [{:base_type :type/Float+ :name "A double number field"+ :link nil+ :field_type [:entity/GenericTable :type/Number]:score 80}]}}bindings)))))(deftest ^:parallel bind-dimensions-select-most-specific-test(testing "When multiple dimensions are candidates the most specific dimension is selected."- (is (= {"Quantity" {:field_type [:entity/GenericTable :type/Quantity],- :score 100,- :matches [{:semantic_type :type/Quantity,- :name "QUANTITY",- :effective_type :type/Integer,- :display_name "Quantity",- :base_type :type/Integer,- :link nil,- :field_type [:entity/GenericTable :type/Quantity],+ (is (= {"Quantity" {:field_type [:entity/GenericTable :type/Quantity]+ :score 100+ :matches [{:semantic_type :type/Quantity+ :name "QUANTITY"+ :effective_type :type/Integer+ :display_name "Quantity"+ :base_type :type/Integer+ :link nil+ :field_type [:entity/GenericTable :type/Quantity]:score 100}]}}(let [context {:source {:entity_type :entity/GenericTable- :fields [{:semantic_type :type/Discount,+ :fields [{:semantic_type :type/Discount:name "DISCOUNT":effective_type :type/Float:base_type :type/Float}- {:semantic_type :type/Quantity,+ {:semantic_type :type/Quantity:name "QUANTITY":effective_type :type/Integer:base_type :type/Integer}]}:tables [{:entity_type :entity/TransactionTable- :fields [{:semantic_type :type/Discount,+ :fields [{:semantic_type :type/Discount:name "DISCOUNT"- :effective_type :type/Float,+ :effective_type :type/Float:display_name "Discount":base_type :type/Float}- {:semantic_type :type/Quantity,+ {:semantic_type :type/Quantity:name "QUANTITY":effective_type :type/Integer:display_name "Quantity"@@ -539,49 +524,80 @@(deftest ^:parallel grounded-metrics-test(mt/dataset test-data- (let [test-metrics [{:metric ["count"], :score 100, :metric-name "Count"}- {:metric ["distinct" ["dimension" "FK"]], :score 100, :metric-name "CountDistinctFKs"}- {:metric ["/"- ["dimension" "Discount"]- ["dimension" "Income"]], :score 100, :metric-name "AvgDiscount"}- {:metric ["sum" ["dimension" "GenericNumber"]], :score 100, :metric-name "Sum"}- {:metric ["avg" ["dimension" "GenericNumber"]], :score 100, :metric-name "Avg"}]- total-field {:id 1 :name "TOTAL"}- discount-field {:id 2 :name "DISCOUNT"}- income-field {:id 3 :name "INCOME"}]+ (let [test-metrics [{:metric ["count"], :score 100, :metric-name "Count"}+ {:metric ["distinct" ["dimension" "FK"]], :score 100, :metric-name "CountDistinctFKs"}+ {:metric ["/"+ ["dimension" "Discount"]+ ["dimension" "Income"]]+ :score 100+ :metric-name "AvgDiscount"}+ {:metric ["sum" ["dimension" "GenericNumber"]], :score 100, :metric-name "Sum"}+ {:metric ["avg" ["dimension" "GenericNumber"]], :score 100, :metric-name "Avg"}]+ total-field (t2/instance :model/Field {:id 1, :name "TOTAL", :base_type :type/Number})+ discount-field (t2/instance :model/Field {:id 2, :name "DISCOUNT", :base_type :type/Number})+ income-field (t2/instance :model/Field {:id 3, :name "INCOME", :base_type :type/Number})](testing "When no dimensions are provided, we produce grounded dimensionless metrics"- (is (= [{:metric-name "Count"- :metric-title "Count"- :metric-score 100- :metric-definition {:aggregation [["count"]]}+ (is (= [{:metric-name "Count"+ :metric-title "Count"+ :metric-score 100+ :metric-definition {:xrays/aggregations [{:lib/type :lib/external-op, :operator :count, :args []}]}:dimension-name->field {}}](interesting/grounded-metricstest-metrics{"Count" {:matches []}}))))(testing "When we can match on a dimension, we produce every matching metric (2 for GenericNumber)"- (is (=? [{:metric-name "Sum",- :metric-definition {:aggregation [["sum" "TOTAL"]]}}- {:metric-name "Avg",- :metric-definition {:aggregation [["avg" "TOTAL"]]}}]+ (is (=? [{:metric-name "Sum"+ :metric-definition {:xrays/aggregations [{:lib/type :lib/external-op+ :operator :sum+ :args [{:lib/type :metadata/column+ :id 1+ :name "TOTAL"+ :base-type :type/Number}]}]}}+ {:metric-name "Avg"+ :metric-definition {:xrays/aggregations [{:lib/type :lib/external-op+ :operator :avg+ :args [{:lib/type :metadata/column+ :id 1+ :name "TOTAL"+ :base-type :type/Number}]}]}}](interesting/grounded-metrics- ;; Drop Count+ ;; Drop Count(rest test-metrics){"Count" {:matches []}"GenericNumber" {:matches [total-field]}}))))(testing "The addition of Discount doesn't add more matches as we needIncome as well to add the metric that uses Discount"(is (=? [{:metric-name "Sum"- :metric-definition {:aggregation [["sum" "TOTAL"]]}}+ :metric-definition {:xrays/aggregations [{:lib/type :lib/external-op+ :operator :sum+ :args [{:lib/type :metadata/column+ :id 1+ :name "TOTAL"+ :base-type :type/Number}]}]}}{:metric-name "Avg"- :metric-definition {:aggregation [["avg" "TOTAL"]]}}]+ :metric-definition {:xrays/aggregations [{:lib/type :lib/external-op+ :operator :avg+ :args [{:lib/type :metadata/column+ :id 1+ :name "TOTAL"+ :base-type :type/Number}]}]}}](interesting/grounded-metrics(rest test-metrics){"Count" {:matches []}"GenericNumber" {:matches [total-field]}"Discount" {:matches [discount-field]}}))))(testing "Discount and Income will add the satisfied AvgDiscount grounded metric"- (is (=? [{:metric-name "AvgDiscount",- :metric-definition {:aggregation [["/" "DISCOUNT" "INCOME"]]}}+ (is (=? [{:metric-name "AvgDiscount"+ :metric-definition {:xrays/aggregations [{:lib/type :lib/external-op+ :operator :/+ :args [{:lib/type :metadata/column+ :id 2+ :name "DISCOUNT"+ :base-type :type/Number}+ {:lib/type :metadata/column+ :id 3+ :name "INCOME"+ :base-type :type/Number}]}]}}{:metric-name "Sum"}{:metric-name "Avg"}](interesting/grounded-metricsdiff --git a/test/metabase/xrays/automagic_dashboards/names_test.clj b/test/metabase/xrays/automagic_dashboards/names_test.cljindex ab44c6685a2..b6aa36b0bdd 100644--- a/test/metabase/xrays/automagic_dashboards/names_test.clj+++ b/test/metabase/xrays/automagic_dashboards/names_test.clj@@ -3,12 +3,15 @@[clojure.string :as str][clojure.test :refer :all][java-time.api :as t]+ [metabase.lib.core :as lib][metabase.test :as mt][metabase.util.date-2 :as u.date][metabase.util.i18n :refer [tru]]+ [metabase.util.malli :as mu][metabase.xrays.api.automagic-dashboards :as api.automagic-dashboards][metabase.xrays.automagic-dashboards.core :as magic]- [metabase.xrays.automagic-dashboards.names :as names]))+ [metabase.xrays.automagic-dashboards.names :as names]+ [metabase.xrays.automagic-dashboards.schema :as ads]));;; ------------------- Datetime humanization (for chart and dashboard titles) -------------------@@ -36,7 +39,9 @@:week-of-year (u.date/extract dt :week-of-year)}](testing (format "unit = %s" unit)(is (= (str expected)- (str (#'names/humanize-datetime t-str unit))))))))+ (str (#'names/humanize-datetime t-str unit)))))))))++(deftest ^:parallel temporal-humanization-test-2(testing "Extracted unit handling"(is (= :sunday (#'u.date/start-of-week)) "adjust the test, it assumes Sunday as first day of the week")(let [t 3] ; t = 2 or t = 4 would also work@@ -68,6 +73,10 @@(is (some? (#'names/humanize-datetime "1990-09-09T12:30:00" unit)))))));;; ------------------- Cell titles -------------------++(defn- cell-title [root cell-query]+ (names/cell-title root (lib/normalize ::ads/root.cell-query cell-query)))+(deftest ^:parallel cell-title-test(mt/$ids venues(let [query (api.automagic-dashboards/adhoc-query-instance {:query {:source-table (mt/id :venues)@@ -78,30 +87,30 @@(testing "Should humanize equal filter"(is (= "number of Venues where Name is Test";; Test specifically the un-normalized form (metabase#15737)- (names/cell-title root ["=" ["field" %name nil] "Test"]))))+ (cell-title root ["=" ["field" %name nil] "Test"]))))(testing "Should humanize greater than filter"(is (= "number of Venues where Name is greater than Test"- (names/cell-title root [">" ["field" %name nil] "Test"]))))+ (cell-title root [">" ["field" %name nil] "Test"]))))(testing "Should humanize at least filter"(is (= "number of Venues where Name is at least Test"- (names/cell-title root [">=" ["field" %name nil] "Test"]))))+ (cell-title root [">=" ["field" %name nil] "Test"]))))(testing "Should humanize less than filter"(is (= "number of Venues where Name is less than Test"- (names/cell-title root ["<" ["field" %name nil] "Test"]))))+ (cell-title root ["<" ["field" %name nil] "Test"]))))(testing "Should humanize no more than filter"(is (= "number of Venues where Name is no more than Test"- (names/cell-title root ["<=" ["field" %name nil] "Test"]))))+ (cell-title root ["<=" ["field" %name nil] "Test"]))))(testing "Should humanize and filter"(is (= "number of Venues where Name is Test and Price is 0"- (names/cell-title root ["and"- ["=" $name "Test"]- ["=" $price 0]]))))+ (cell-title root ["and"+ ["=" $name "Test"]+ ["=" $price 0]]))))(testing "Should humanize between filter"(is (= "number of Venues where Name is between A and J"- (names/cell-title root ["between" $name "A", "J"]))))+ (cell-title root ["between" $name "A", "J"]))))(testing "Should humanize inside filter"(is (= "number of Venues where Longitude is between 2 and 4; and Latitude is between 3 and 1"- (names/cell-title root ["inside" (mt/$ids venues $latitude) (mt/$ids venues $longitude) 1 2 3 4])))))))+ (cell-title root ["inside" (mt/$ids venues $latitude) (mt/$ids venues $longitude) 1 2 3 4])))))))(deftest ^:parallel cell-title-default-test(mt/$ids venues@@ -111,8 +120,9 @@:database (mt/id)})root (magic/->root query)](testing "Just say \"relates to\" when we don't know what the operator is"- (is (= "number of Venues where Name relates to Test"- (names/cell-title root ["???" ["field" %name nil] "Test"])))))))+ (mu/disable-enforcement+ (is (= "number of Venues where Name relates to Test"+ (cell-title root ["???" [:field {:lib/uuid (str (random-uuid))} %name] "Test"]))))))))(deftest ^:parallel cell-title-with-dates-comparison-test(testing "Ensure cell titles with date comparisons display correctly"@@ -126,24 +136,24 @@;; humanize-datetime which we probably should not do right now.(testing "Should humanize temporal field with = test"(is (= "number of Users where Last Login is on September 9, 1990"- (names/cell-title root- ["=" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))+ (cell-title root+ ["=" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))(testing "Should humanize temporal field with > test"(is (= "number of Users where Last Login is after on September 9, 1990"- (names/cell-title root- [">" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))+ (cell-title root+ [">" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))(testing "Should humanize temporal field with < test"(is (= "number of Users where Last Login is before on September 9, 1990"- (names/cell-title root- ["<" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))+ (cell-title root+ ["<" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))(testing "Should humanize temporal field with >= test"(is (= "number of Users where Last Login is not before on September 9, 1990"- (names/cell-title root- [">=" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))+ (cell-title root+ [">=" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))(testing "Should humanize temporal field with <= test"(is (= "number of Users where Last Login is not after on September 9, 1990"- (names/cell-title root- ["<=" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))))))+ (cell-title root+ ["<=" ["field" %last_login {:temporal-unit :day}] "1990-09-09T12:30:00"]))))))))(deftest ^:parallel no-null-cell-title-temporal-units-35170-test(testing "Ensure various permutations of temporal units produce valid strings (without nulls in them)@@ -156,25 +166,25 @@root (magic/->root query)](testing "Should not contain nulls when temporal-unit is hour"(is (= "number of Users where Last Login is at 12 PM, September 9, 1990"- (names/cell-title root- ["=" ["field" %last_login {:temporal-unit :hour}] "1990-09-09T12:30:00"]))))+ (cell-title root+ ["=" ["field" %last_login {:temporal-unit :hour}] "1990-09-09T12:30:00"]))))(testing "Should not contain nulls when temporal-unit is day"(is (= "number of Users where Last Login is on January 1, 2020"- (names/cell-title root- ["=" ["field" %last_login {:temporal-unit :day}] "2020"]))))+ (cell-title root+ ["=" ["field" %last_login {:temporal-unit :day}] "2020"]))))(testing "Should not contain nulls when temporal-unit is month"(is (= "number of Users where Last Login is in January 2020"- (names/cell-title root- ["=" ["field" %last_login {:temporal-unit :month}] "2020"]))))+ (cell-title root+ ["=" ["field" %last_login {:temporal-unit :month}] "2020"]))))(testing "Should not contain nulls when temporal-unit is quarter"(is (= "number of Users where Last Login is in Q1 - 2020"- (names/cell-title root- ["=" ["field" %last_login {:temporal-unit :quarter}] "2020"]))))+ (cell-title root+ ["=" ["field" %last_login {:temporal-unit :quarter}] "2020"]))));; This was producing "number of Users where null of Last Login is 2020" originally(testing "Should not contain nulls when temporal-unit is year"(is (= "number of Users where year of Last Login is 2020"- (names/cell-title root- ["=" ["field" %last_login {:temporal-unit :year}] "2020"]))))))))+ (cell-title root+ ["=" ["field" %last_login {:temporal-unit :year}] "2020"]))))))))(deftest ^:parallel humanize-filter-value-works-for-expressions-16680-test(testing "Expressions can be used in humanized names in addition to fields"@@ -182,8 +192,12 @@(is (= "TestColumn is 2 and Created At is in February 2024"(#'names/humanize-filter-value{:database (mt/id)}- ["and"- ["=" ["expression" "TestColumn" {:base-type "type/Integer"}] 2]- ["=" ["field" (mt/id :orders :created_at)- {:base-type "type/DateTime", :temporal-unit "month"}]+ [:and+ {:lib/uuid "433ee131-2210-4b94-88fb-ff4967eaa555"}+ [:= {:lib/uuid "02a0746d-cf2d-4878-b5e6-26bce3e81986"}+ [:expression {:base-type :type/Integer, :lib/uuid "9ec2cd31-02b9-404d-a63b-681a140b9498"} "TestColumn"]+ 2]+ [:= {:lib/uuid "88ddba2f-5047-4b31-9c7f-6c0b5bc8ffd9"}+ [:field {:base-type :type/DateTime, :temporal-unit :month, :lib/uuid "c40000a9-da61-410d-8822-0b888fd2396a"}+ (mt/id :orders :created_at)]"2024-02-01T00:00:00Z"]]))))))diff --git a/test/metabase/xrays/automagic_dashboards/schema_test.clj b/test/metabase/xrays/automagic_dashboards/schema_test.cljnew file mode 100644index 00000000000..bb48789d3ba--- /dev/null+++ b/test/metabase/xrays/automagic_dashboards/schema_test.clj@@ -0,0 +1,26 @@+(ns metabase.xrays.automagic-dashboards.schema-test+ (:require+ [clojure.test :refer :all]+ [metabase.lib.core :as lib]+ [metabase.xrays.automagic-dashboards.schema :as ads]))++(deftest ^:parallel normalize-cell-query-test+ (is (=? [:=+ {:lib/uuid string?}+ [:field {:base-type :type/Text, :lib/uuid string?} "SOURCE"]+ "Affiliate"]+ (lib/normalize ::ads/root.cell-query ["=" ["field" "SOURCE" {"base-type" "type/Text"}] "Affiliate"])))+ (is (=? [:and+ {:lib/uuid string?}+ [:= {:lib/uuid string?}+ [:expression {:lib/uuid string?} "TestColumn"]+ 2]+ [:= {:lib/uuid string?}+ [:field {:temporal-unit :month, :lib/uuid string?}+ 13]+ "2019-02-01T00:00:00Z"]]+ (lib/normalize ::ads/root.cell-query [:and+ [:= [:expression "TestColumn"] 2]+ [:=+ [:field 13 {:temporal-unit :month}]+ "2019-02-01T00:00:00Z"]]))))diff --git a/test/metabase/xrays/automagic_dashboards/util_test.clj b/test/metabase/xrays/automagic_dashboards/util_test.cljindex 2c0683a2aff..cddcc9b4d45 100644--- a/test/metabase/xrays/automagic_dashboards/util_test.clj+++ b/test/metabase/xrays/automagic_dashboards/util_test.clj@@ -19,7 +19,9 @@(is (=? {:id (mt/id :orders :discount)}(magic.util/->field root (mt/id :orders :discount))))(is (=? {:id (mt/id :orders :discount)}- (magic.util/->field root [:field (mt/id :orders :discount) nil]))))))))))+ (magic.util/->field+ root+ [:field {:lib/uuid (str (random-uuid))} (mt/id :orders :discount)]))))))))))(deftest ^:parallel ->field-test-2(testing "Demonstrate the stated methods in which ->fields works"@@ -33,7 +35,9 @@(is (=? {:id (mt/id :orders :discount)}(magic.util/->field root (mt/id :orders :discount))))(is (=? {:id (mt/id :orders :discount)}- (magic.util/->field root [:field (mt/id :orders :discount) nil]))))+ (magic.util/->field+ root+ [:field {:lib/uuid (str (random-uuid))} (mt/id :orders :discount)]))))(testing "Looking up the field by name or named field ref works,returning the metadata description of the field."(is (=? {:name "DISCOUNT"@@ -41,4 +45,6 @@(magic.util/->field root "DISCOUNT"))))(is (=? {:name "DISCOUNT":field_ref [:field "DISCOUNT" {:base-type :type/Float}]}- (magic.util/->field root [:field "DISCOUNT" {:base-type :type/Float}])))))))))))+ (magic.util/->field+ root+ [:field {:base-type :type/Float, :lib/uuid (str (random-uuid))} "DISCOUNT"])))))))))))