Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Support measures and segments for Metabot (#67368)

Thomas Schmidt5ab27ac0fbc4
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj
index eae3eb37869..c6f792b7b1f 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj
@@ -163,8 +163,16 @@
[:values [:sequential [:or :int :double]]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
+(mr/def ::segment-filter
+ "Filter using a pre-defined segment."
+ [:and
+ [:map
+ [:segment_id :int]]
+ [:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
+
(mr/def ::filter
[:or
+ ::segment-filter
::existence-filter
::temporal-extraction-filter ::disjunctive-temporal-extraction-filter
::temporal-filter ::disjunctive-temporal-filter
@@ -180,7 +188,8 @@
"minute", "hour" "day" "week" "month" "quarter" "year" "day-of-week"]]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
-(mr/def ::aggregation
+(mr/def ::field-aggregation
+ "Aggregation using a field and function."
[:and
[:map
[:field_id :string]
@@ -190,6 +199,18 @@
"avg" "count" "count-distinct" "max" "min" "sum"]]]
[:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
+(mr/def ::measure-aggregation
+ "Aggregation using a pre-defined measure."
+ [:and
+ [:map
+ [:measure_id :int]
+ [:sort_order {:optional true} [:maybe [:enum {:encode/tool-api-request keyword} "asc" "desc"]]]]
+ [:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])
+
+(mr/def ::aggregation
+ "Aggregation - either field-based or measure-based."
+ [:or ::field-aggregation ::measure-aggregation])
+
(mr/def ::field
[:and
[:map
@@ -282,8 +303,25 @@
::basic-metric
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:queryable_dimensions {:optional true} ::columns]
+ [:segments {:optional true} [:sequential ::segment]]
[:verified {:optional true} :boolean]]])
+(mr/def ::measure
+ [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
+ [:id :int]
+ [:name :string]
+ [:display_name {:optional true} [:maybe :string]]
+ [:description {:optional true} [:maybe :string]]
+ [:definition {:optional true} [:maybe :map]]])
+
+(mr/def ::segment
+ [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
+ [:id :int]
+ [:name :string]
+ [:display_name {:optional true} [:maybe :string]]
+ [:description {:optional true} [:maybe :string]]
+ [:definition {:optional true} [:maybe :map]]])
+
(mr/def ::table-result
[:map
[:id :int]
@@ -297,7 +335,9 @@
[:related_tables {:optional true} [:sequential [:ref ::table-result]]]
[:related_by {:optional true} [:maybe :string]]
[:description {:optional true} [:maybe :string]]
- [:metrics {:optional true} [:sequential ::basic-metric]]])
+ [:metrics {:optional true} [:sequential ::basic-metric]]
+ [:measures {:optional true} [:sequential ::measure]]
+ [:segments {:optional true} [:sequential ::segment]]])
(mr/def ::basic-transform
[:map
@@ -590,12 +630,14 @@
[:metric_id :int]
[:with_default_temporal_breakout {:optional true, :default true} :boolean]
[:with_field_values {:optional true, :default true} :boolean]
- [:with_queryable_dimensions {:optional true, :default true} :boolean]]
+ [:with_queryable_dimensions {:optional true, :default true} :boolean]
+ [:with_segments {:optional true, :default false} :boolean]]
[:map {:encode/tool-api-request
#(set/rename-keys % {:metric_id :metric-id
:with_default_temporal_breakout :with-default-temporal-breakout?
:with_field_values :with-field-values?
- :with_queryable_dimensions :with-queryable-dimensions?})}]])
+ :with_queryable_dimensions :with-queryable-dimensions?
+ :with_segments :with-segments?})}]])
(mr/def ::get-metric-details-result
[:or
@@ -694,7 +736,9 @@
[:with_field_values {:optional true, :default true} :boolean]
[:with_related_tables {:optional true, :default true} :boolean]
[:with_metrics {:optional true, :default true} :boolean]
- [:with_metric_default_temporal_breakout {:optional true, :default true} :boolean]]
+ [:with_metric_default_temporal_breakout {:optional true, :default true} :boolean]
+ [:with_measures {:optional true, :default false} :boolean]
+ [:with_segments {:optional true, :default false} :boolean]]
[:fn {:error/message "Exactly one of model_id and table_id required"}
#(= (count (select-keys % [:model_id :table_id])) 1)]
[:map {:encode/tool-api-request
@@ -704,7 +748,9 @@
:with_field_values :with-field-values?
:with_related_tables :with-related-tables?
:with_metrics :with-metrics?
- :with_metric_default_temporal_breakout :with-default-temporal-breakout?})}]])
+ :with_metric_default_temporal_breakout :with-default-temporal-breakout?
+ :with_measures :with-measures?
+ :with_segments :with-segments?})}]])
(mr/def ::get-table-details-result
[:or
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/entity_details.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/entity_details.clj
index ebc214bbc90..1b3bb01f7ce 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/entity_details.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/entity_details.clj
@@ -18,6 +18,11 @@
(set! *warn-on-reflection* true)
+(defn- convert-measure-or-segment
+ "Convert a measure or segment metadata object to the format expected by the API."
+ [metadata]
+ (select-keys metadata [:id :name :display-name :description :definition]))
+
(defn verified-review?
"Return true if the most recent ModerationReview for the given item id/type is verified."
[id item-type]
@@ -94,12 +99,14 @@
([id options]
(when-let [card (metabot-v3.tools.u/get-card id)]
(metric-details card (lib-be/application-database-metadata-provider (:database_id card)) options)))
- ([card metadata-provider {:keys [field-values-fn with-default-temporal-breakout? with-queryable-dimensions?]
+ ([card metadata-provider {:keys [field-values-fn with-default-temporal-breakout? with-queryable-dimensions?
+ with-segments?]
:or {field-values-fn add-field-values
with-default-temporal-breakout? true
- with-queryable-dimensions? true}}]
+ with-queryable-dimensions? true
+ with-segments? false}}]
(let [id (:id card)
- query-needed? (or with-default-temporal-breakout? with-queryable-dimensions?)
+ query-needed? (or with-default-temporal-breakout? with-queryable-dimensions? with-segments?)
metric-query (when query-needed?
(lib/query metadata-provider (lib.metadata/card metadata-provider id)))
breakouts (when query-needed?
@@ -136,7 +143,12 @@
(map #(metabot-v3.tools.u/->result-column
metric-query % (col-index %) field-id-prefix)))
(->> (lib/filterable-columns base-query)
- field-values-fn)))))))
+ field-values-fn)))
+
+ with-segments?
+ (assoc :segments (if-let [segments (lib/available-segments metric-query)]
+ (mapv convert-measure-or-segment segments)
+ []))))))
(defn- convert-metric
([db-metric metadata-provider]
@@ -150,16 +162,19 @@
(defn- table-details
([id] (table-details id nil))
- ([id {:keys [metadata-provider field-values-fn with-fields? with-related-tables? with-metrics?]
+ ([id {:keys [metadata-provider field-values-fn with-fields? with-related-tables? with-metrics?
+ with-measures? with-segments?]
:or {field-values-fn add-field-values
with-fields? true
with-related-tables? true
- with-metrics? true}
+ with-metrics? true
+ with-measures? false
+ with-segments? false}
:as options}]
(when-let [base (if metadata-provider
(lib.metadata/table metadata-provider id)
(metabot-v3.tools.u/get-table id :db_id :description :name :schema))]
- (let [query-needed? (or with-fields? with-related-tables? with-metrics?)
+ (let [query-needed? (or with-fields? with-related-tables? with-metrics? with-measures? with-segments?)
db-id (if metadata-provider (:db-id base) (:db_id base))
db-engine (:engine (if metadata-provider
(lib.metadata/database metadata-provider)
@@ -191,7 +206,13 @@
:related_tables related-tables
:metrics (when with-metrics?
(not-empty (mapv #(convert-metric % mp options)
- (lib/available-metrics table-query))))))))))
+ (lib/available-metrics table-query))))
+ :measures (when with-measures?
+ (not-empty (mapv convert-measure-or-segment
+ (lib/available-measures table-query))))
+ :segments (when with-segments?
+ (not-empty (mapv convert-measure-or-segment
+ (lib/available-segments table-query))))))))))
(defn related-tables
"Constructs a list of tables, optionally including their fields, that are related to the given query via foreign key.
@@ -236,18 +257,21 @@
([id options]
(when-let [card (metabot-v3.tools.u/get-card id)]
(card-details card (lib-be/application-database-metadata-provider (:database_id card)) options)))
- ([base metadata-provider {:keys [field-values-fn with-fields? with-related-tables? with-metrics?]
+ ([base metadata-provider {:keys [field-values-fn with-fields? with-related-tables? with-metrics?
+ with-measures? with-segments?]
:or {field-values-fn add-field-values
with-fields? true
with-related-tables? true
- with-metrics? true}
+ with-metrics? true
+ with-measures? false
+ with-segments? false}
:as options}]
(let [id (:id base)
database-id (:database_id base)
database-engine (:engine (lib.metadata/database metadata-provider))
card-metadata (lib.metadata/card metadata-provider id)
dataset-query (get card-metadata :dataset-query)
- query-needed? (or with-fields? with-related-tables? with-metrics?)
+ query-needed? (or with-fields? with-related-tables? with-metrics? with-measures? with-segments?)
;; pivot questions have strange result-columns so we work with the dataset-query
card-type (:type base)
card-query (when query-needed?
@@ -276,7 +300,13 @@
:related_tables related-tables
:metrics (when with-metrics?
(not-empty (mapv #(convert-metric % metadata-provider options)
- (lib/available-metrics card-query)))))))))
+ (lib/available-metrics card-query))))
+ :measures (when with-measures?
+ (not-empty (mapv convert-measure-or-segment
+ (lib/available-measures card-query))))
+ :segments (when with-segments?
+ (not-empty (mapv convert-measure-or-segment
+ (lib/available-segments card-query)))))))))
(defn cards-details
"Get the details of metrics or models as specified by `card-type` and `cards`
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/filters.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/filters.clj
index 5ab3dd3a8a5..bd8f07ff93b 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/filters.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/filters.clj
@@ -8,7 +8,8 @@
[metabase.lib.options :as lib.options]
[metabase.lib.types.isa :as lib.types.isa]
[metabase.lib.util :as lib.util]
- [metabase.util :as u]))
+ [metabase.util :as u]
+ [metabase.util.i18n :refer [tru]]))
(defn- apply-filter-bucket
[column bucket]
@@ -76,66 +77,74 @@
(defn- add-filter
[query llm-filter]
- (let [{:keys [operation value values]} llm-filter
- expr (filter-bucketed-column llm-filter)
- with-values-or-value (fn with-values-or-value
- ([f]
- (with-values-or-value f expr))
- ([f expr]
- (if values
- (apply f expr values)
- (f expr value))))
- string-match (fn [match-fn]
- (-> (with-values-or-value match-fn)
- (lib.options/update-options assoc :case-sensitive false)))
- filter
- (case operation
- :is-null (lib/is-null expr)
- :is-not-null (lib/not-null expr)
- :string-is-empty (lib/is-empty expr)
- :string-is-not-empty (lib/not-empty expr)
- :is-true (lib/= expr true)
- :is-false (lib/= expr false)
- :equals (with-values-or-value lib/=)
- :not-equals (with-values-or-value lib/!=)
- :greater-than (lib/> expr value)
- :greater-than-or-equal (lib/>= expr value)
- :less-than (lib/< expr value)
- :less-than-or-equal (lib/<= expr value)
- :year-equals (with-values-or-value lib/= (lib/get-year expr))
- :year-not-equals (with-values-or-value lib/!= (lib/get-year expr))
- :quarter-equals (with-values-or-value lib/= (lib/get-quarter expr))
- :quarter-not-equals (with-values-or-value lib/!= (lib/get-quarter expr))
- :month-equals (with-values-or-value lib/= (lib/get-month expr))
- :month-not-equals (with-values-or-value lib/!= (lib/get-month expr))
- :day-of-week-equals (with-values-or-value lib/= (lib/get-day-of-week expr :iso))
- :day-of-week-not-equals (with-values-or-value lib/!= (lib/get-day-of-week expr :iso))
- :hour-equals (with-values-or-value lib/= (lib/get-hour expr))
- :hour-not-equals (with-values-or-value lib/!= (lib/get-hour expr))
- :minute-equals (with-values-or-value lib/= (lib/get-minute expr))
- :minute-not-equals (with-values-or-value lib/!= (lib/get-minute expr))
- :second-equals (with-values-or-value lib/= (lib/get-second expr))
- :second-not-equals (with-values-or-value lib/!= (lib/get-second expr))
- :date-equals (with-values-or-value lib/=)
- :date-not-equals (with-values-or-value lib/!=)
- :date-before (lib/< expr value)
- :date-on-or-before (lib/<= expr value)
- :date-after (lib/> expr value)
- :date-on-or-after (lib/>= expr value)
- :string-equals (with-values-or-value lib/=)
- :string-not-equals (with-values-or-value lib/!=)
- :string-contains (string-match lib/contains)
- :string-not-contains (string-match lib/does-not-contain)
- :string-starts-with (string-match lib/starts-with)
- :string-ends-with (string-match lib/ends-with)
- :number-equals (with-values-or-value lib/=)
- :number-not-equals (with-values-or-value lib/!=)
- :number-greater-than (lib/> expr value)
- :number-greater-than-or-equal (lib/>= expr value)
- :number-less-than (lib/< expr value)
- :number-less-than-or-equal (lib/<= expr value)
- (throw (ex-info (str "unknown filter operation " operation) {:agent-error? true})))]
- (lib/filter query filter)))
+ (if-let [segment-id (:segment-id llm-filter)]
+ ;; Segment-based filter
+ (if-let [segment (lib.metadata/segment query segment-id)]
+ (lib/filter query segment)
+ (throw (ex-info (tru "Segment with id {0} not found" segment-id)
+ {:agent-error? true
+ :segment-id segment-id})))
+ ;; Standard field-based filter logic
+ (let [{:keys [operation value values]} llm-filter
+ expr (filter-bucketed-column llm-filter)
+ with-values-or-value (fn with-values-or-value
+ ([f]
+ (with-values-or-value f expr))
+ ([f expr]
+ (if values
+ (apply f expr values)
+ (f expr value))))
+ string-match (fn [match-fn]
+ (-> (with-values-or-value match-fn)
+ (lib.options/update-options assoc :case-sensitive false)))
+ filter
+ (case operation
+ :is-null (lib/is-null expr)
+ :is-not-null (lib/not-null expr)
+ :string-is-empty (lib/is-empty expr)
+ :string-is-not-empty (lib/not-empty expr)
+ :is-true (lib/= expr true)
+ :is-false (lib/= expr false)
+ :equals (with-values-or-value lib/=)
+ :not-equals (with-values-or-value lib/!=)
+ :greater-than (lib/> expr value)
+ :greater-than-or-equal (lib/>= expr value)
+ :less-than (lib/< expr value)
+ :less-than-or-equal (lib/<= expr value)
+ :year-equals (with-values-or-value lib/= (lib/get-year expr))
+ :year-not-equals (with-values-or-value lib/!= (lib/get-year expr))
+ :quarter-equals (with-values-or-value lib/= (lib/get-quarter expr))
+ :quarter-not-equals (with-values-or-value lib/!= (lib/get-quarter expr))
+ :month-equals (with-values-or-value lib/= (lib/get-month expr))
+ :month-not-equals (with-values-or-value lib/!= (lib/get-month expr))
+ :day-of-week-equals (with-values-or-value lib/= (lib/get-day-of-week expr :iso))
+ :day-of-week-not-equals (with-values-or-value lib/!= (lib/get-day-of-week expr :iso))
+ :hour-equals (with-values-or-value lib/= (lib/get-hour expr))
+ :hour-not-equals (with-values-or-value lib/!= (lib/get-hour expr))
+ :minute-equals (with-values-or-value lib/= (lib/get-minute expr))
+ :minute-not-equals (with-values-or-value lib/!= (lib/get-minute expr))
+ :second-equals (with-values-or-value lib/= (lib/get-second expr))
+ :second-not-equals (with-values-or-value lib/!= (lib/get-second expr))
+ :date-equals (with-values-or-value lib/=)
+ :date-not-equals (with-values-or-value lib/!=)
+ :date-before (lib/< expr value)
+ :date-on-or-before (lib/<= expr value)
+ :date-after (lib/> expr value)
+ :date-on-or-after (lib/>= expr value)
+ :string-equals (with-values-or-value lib/=)
+ :string-not-equals (with-values-or-value lib/!=)
+ :string-contains (string-match lib/contains)
+ :string-not-contains (string-match lib/does-not-contain)
+ :string-starts-with (string-match lib/starts-with)
+ :string-ends-with (string-match lib/ends-with)
+ :number-equals (with-values-or-value lib/=)
+ :number-not-equals (with-values-or-value lib/!=)
+ :number-greater-than (lib/> expr value)
+ :number-greater-than-or-equal (lib/>= expr value)
+ :number-less-than (lib/< expr value)
+ :number-less-than-or-equal (lib/<= expr value)
+ (throw (ex-info (str "unknown filter operation " operation) {:agent-error? true})))]
+ (lib/filter query filter))))
(defn- add-breakout
[query {:keys [column field-granularity]}]
@@ -146,17 +155,18 @@
(lib/breakout query expr)))
(defn- query-metric*
- [{:keys [metric-id filters group-by]}]
+ [{:keys [metric-id filters group-by] :as _arguments}]
(let [card (metabot-v3.tools.u/get-card metric-id)
mp (lib-be/application-database-metadata-provider (:database_id card))
base-query (->> (lib/query mp (lib.metadata/card mp metric-id))
lib/remove-all-breakouts)
field-id-prefix (metabot-v3.tools.u/card-field-id-prefix metric-id)
visible-cols (lib/visible-columns base-query)
+ resolve-visible-column #(metabot-v3.tools.u/resolve-column % field-id-prefix visible-cols)
+ ;; Separate segment filters from field filters before column resolution
+ resolved-filters (map #(if (:segment-id %) % (resolve-visible-column %)) filters)
query (as-> base-query $q
- (reduce add-filter
- $q
- (map #(metabot-v3.tools.u/resolve-column % field-id-prefix visible-cols) filters))
+ (reduce add-filter $q resolved-filters)
(reduce add-breakout
$q
(map #(metabot-v3.tools.u/resolve-column % field-id-prefix visible-cols) group-by)))
@@ -165,8 +175,7 @@
returned-cols (lib/returned-columns query)]
{:type :query
:query-id query-id
- ;; existing usage, don't do this going forward -- use Lib instead and persist MBQL 5 to the app DB
- :query #_{:clj-kondo/ignore [:discouraged-var]} (lib/->legacy-MBQL query)
+ :query query
:result-columns (into []
(map-indexed #(metabot-v3.tools.u/->result-column query %2 %1 query-field-id-prefix))
returned-cols)}))
@@ -183,23 +192,37 @@
{:output (str "No metric found with metric_id " metric-id)}
(metabot-v3.tools.u/handle-agent-error e)))))
+(defn- apply-aggregation-sort-order
+ "If sort-order is specified, add an order-by clause for the last aggregation in the query."
+ [query sort-order]
+ (if sort-order
+ (let [query-aggregations (lib/aggregations query)
+ last-aggregation-idx (dec (count query-aggregations))]
+ (lib/order-by query (lib/aggregation-ref query last-aggregation-idx) sort-order))
+ query))
+
(defn- add-aggregation
[query aggregation]
- (let [expr (bucketed-column aggregation)
- sort-order (:sort-order aggregation)
- agg-expr (case (:function aggregation)
- :count (lib/count)
- :count-distinct (lib/distinct expr)
- :sum (lib/sum expr)
- :min (lib/min expr)
- :max (lib/max expr)
- :avg (lib/avg expr))
- query-with-aggregation (lib/aggregate query agg-expr)]
- (if sort-order
- (let [query-aggregations (lib/aggregations query-with-aggregation)
- last-aggregation-idx (dec (count query-aggregations))]
- (lib/order-by query-with-aggregation (lib/aggregation-ref query-with-aggregation last-aggregation-idx) sort-order))
- query-with-aggregation)))
+ (let [sort-order (:sort-order aggregation)
+ query-with-aggregation
+ (if-let [measure-id (:measure-id aggregation)]
+ ;; Measure-based aggregation
+ (if-let [measure (lib.metadata/measure query measure-id)]
+ (lib/aggregate query measure)
+ (throw (ex-info (tru "Measure with id {0} not found" measure-id)
+ {:agent-error? true
+ :measure-id measure-id})))
+ ;; Field-based aggregation
+ (let [expr (bucketed-column aggregation)
+ agg-expr (case (:function aggregation)
+ :count (lib/count)
+ :count-distinct (lib/distinct expr)
+ :sum (lib/sum expr)
+ :min (lib/min expr)
+ :max (lib/max expr)
+ :avg (lib/avg expr))]
+ (lib/aggregate query agg-expr)))]
+ (apply-aggregation-sort-order query-with-aggregation sort-order)))
(defn- expression?
[expr-or-column]
@@ -219,16 +242,18 @@
(lib/order-by query (:column field) direction))
(defn- add-limit [query limit]
- (cond-> query limit (lib/limit limit)))
+ (if limit
+ (lib/limit query limit)
+ query))
(defn- query-model*
- [{:keys [model-id fields filters aggregations group-by order-by limit]}]
+ [{:keys [model-id fields filters aggregations group-by order-by limit] :as _arguments}]
(let [card (metabot-v3.tools.u/get-card model-id)
mp (lib-be/application-database-metadata-provider (:database_id card))
base-query (lib/query mp (lib.metadata/card mp model-id))
field-id-prefix (metabot-v3.tools.u/card-field-id-prefix model-id)
visible-cols (lib/visible-columns base-query)
- resolve-visible-column #(metabot-v3.tools.u/resolve-column % field-id-prefix visible-cols)
+ resolve-visible-column #(metabot-v3.tools.u/resolve-column % field-id-prefix visible-cols)
resolve-order-by-column (fn [{:keys [field direction]}] {:field (resolve-visible-column field) :direction direction})
projection (map (comp (juxt filter-bucketed-column (fn [{:keys [column bucket]}]
(let [column (cond-> column
@@ -236,24 +261,26 @@
(lib/display-name base-query -1 column :long))))
resolve-visible-column)
fields)
- query (as-> base-query query
- (reduce (fn [query [expr-or-column expr-name]]
- (lib/expression query expr-name expr-or-column))
- query
- (filter (comp expression? first) projection))
- (add-fields query projection)
- (reduce add-filter query (map resolve-visible-column filters))
- (reduce add-aggregation query (map resolve-visible-column aggregations))
- (reduce add-breakout query (map resolve-visible-column group-by))
- (reduce add-order-by query (map resolve-order-by-column order-by))
- (add-limit query limit))
+ ;; Measures and segments don't require column resolution
+ resolved-aggregations (map #(if (:measure-id %) % (resolve-visible-column %)) aggregations)
+ resolved-filters (map #(if (:segment-id %) % (resolve-visible-column %)) filters)
+ reduce-query (fn [query f coll] (reduce f query coll))
+ query (-> base-query
+ (reduce-query (fn [query [expr-or-column expr-name]]
+ (lib/expression query expr-name expr-or-column))
+ (filter (comp expression? first) projection))
+ (add-fields projection)
+ (reduce-query add-filter resolved-filters)
+ (reduce-query add-aggregation resolved-aggregations)
+ (reduce-query add-breakout (map resolve-visible-column group-by))
+ (reduce-query add-order-by (map resolve-order-by-column order-by))
+ (add-limit limit))
query-id (u/generate-nano-id)
query-field-id-prefix (metabot-v3.tools.u/query-field-id-prefix query-id)
returned-cols (lib/returned-columns query)]
{:type :query
:query-id query-id
- ;; existing usage, don't do this going forward -- use Lib instead and persist MBQL 5 to the app DB
- :query #_{:clj-kondo/ignore [:discouraged-var]} (lib/->legacy-MBQL query)
+ :query query
:result-columns (into []
(map-indexed #(metabot-v3.tools.u/->result-column query %2 %1 query-field-id-prefix))
returned-cols)}))
@@ -288,7 +315,7 @@
[{:keys [fields filters aggregations group-by order-by limit] :as arguments}]
(let [[filter-field-id-prefix base-query] (resolve-datasource arguments)
visible-cols (lib/visible-columns base-query)
- resolve-visible-column #(metabot-v3.tools.u/resolve-column % filter-field-id-prefix visible-cols)
+ resolve-visible-column #(metabot-v3.tools.u/resolve-column % filter-field-id-prefix visible-cols)
resolve-order-by-column (fn [{:keys [field direction]}] {:field (resolve-visible-column field) :direction direction})
projection (map (comp (juxt filter-bucketed-column (fn [{:keys [column bucket]}]
(let [column (cond-> column
@@ -296,24 +323,26 @@
(lib/display-name base-query -1 column :long))))
resolve-visible-column)
fields)
- query (as-> base-query query
- (reduce (fn [query [expr-or-column expr-name]]
- (lib/expression query expr-name expr-or-column))
- query
- (filter (comp expression? first) projection))
- (add-fields query projection)
- (reduce add-filter query (map resolve-visible-column filters))
- (reduce add-aggregation query (map resolve-visible-column aggregations))
- (reduce add-breakout query (map resolve-visible-column group-by))
- (reduce add-order-by query (map resolve-order-by-column order-by))
- (add-limit query limit))
+ ;; Measures and segments don't require column resolution
+ all-aggregations (map #(if (:measure-id %) % (resolve-visible-column %)) aggregations)
+ all-filters (map #(if (:segment-id %) % (resolve-visible-column %)) filters)
+ reduce-query (fn [query f coll] (reduce f query coll))
+ query (-> base-query
+ (reduce-query (fn [query [expr-or-column expr-name]]
+ (lib/expression query expr-name expr-or-column))
+ (filter (comp expression? first) projection))
+ (add-fields projection)
+ (reduce-query add-filter all-filters)
+ (reduce-query add-aggregation all-aggregations)
+ (reduce-query add-breakout (map resolve-visible-column group-by))
+ (reduce-query add-order-by (map resolve-order-by-column order-by))
+ (add-limit limit))
query-id (u/generate-nano-id)
query-field-id-prefix (metabot-v3.tools.u/query-field-id-prefix query-id)
returned-cols (lib/returned-columns query)]
{:type :query
:query-id query-id
- ;; existing usage, don't do this going forward -- use Lib instead and persist MBQL 5 to the app DB
- :query #_{:clj-kondo/ignore [:discouraged-var]} (lib/->legacy-MBQL query)
+ :query query
:result-columns (into []
(map-indexed #(metabot-v3.tools.u/->result-column query %2 %1 query-field-id-prefix))
returned-cols)}))
@@ -380,7 +409,7 @@
(defn filter-records
"Add `filters` to the query referenced by `data-source`"
- [{:keys [data-source filters]}]
+ [{:keys [data-source filters] :as _arguments}]
(try
(let [[filter-field-id-prefix base] (base-query data-source)
returned-cols (lib/returned-columns base)
@@ -390,8 +419,7 @@
{:structured-output
{:type :query
:query-id query-id
- ;; existing usage, don't do this going forward -- use Lib instead and persist MBQL 5 to the app DB
- :query #_{:clj-kondo/ignore [:discouraged-var]} (lib/->legacy-MBQL query)
+ :query query
:result-columns (into []
(map-indexed #(metabot-v3.tools.u/->result-column query %2 %1 query-field-id-prefix))
(lib/returned-columns query))}})
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
index 60f1a576316..ffa116ac318 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
@@ -12,12 +12,13 @@
[metabase-enterprise.metabot-v3.tools.filters :as metabot-v3.tools.filters]
[metabase-enterprise.metabot-v3.tools.find-outliers :as metabot-v3.tools.find-outliers]
[metabase-enterprise.metabot-v3.tools.generate-insights :as metabot-v3.tools.generate-insights]
+ [metabase-enterprise.metabot-v3.tools.test-util :as tools.tu]
[metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u]
[metabase-enterprise.metabot-v3.util :as metabot-v3.u]
- [metabase.legacy-mbql.normalize :as mbql.normalize]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.options :as lib.options]
[metabase.test :as mt]
[metabase.util :as u]
[metabase.util.malli.registry :as mr]
@@ -249,21 +250,26 @@
:filters filters
:group_by breakouts}
:conversation_id conversation-id})
- query-id (-> response :structured_output :query_id)]
+ query-id (-> response :structured_output :query_id)
+ actual-query (-> response :structured_output :query lib-be/normalize-query)
+ id-col (lib.metadata/field mp (mt/id :products :id))
+ title-col (lib.metadata/field mp (mt/id :products :title))
+ rating-col (lib.metadata/field mp (mt/id :products :rating))
+ created-at-col (lib.metadata/field mp (mt/id :products :created_at))
+ expected-query (-> (lib/query mp (lib.metadata/table mp (mt/id :products)))
+ (lib/aggregate (lib.options/ensure-uuid [:metric {} metric-id]))
+ (lib/breakout (lib/with-temporal-bucket created-at-col :week))
+ (lib/breakout (lib/with-temporal-bucket created-at-col :day))
+ (lib/filter (lib/> id-col 50))
+ (lib/filter (lib/= title-col "3" "4"))
+ (lib/filter (lib/!= rating-col 3 4))
+ (lib/filter (lib/= (lib/get-month created-at-col) 4 5 9))
+ (lib/filter (lib/!= (lib/get-day created-at-col) 14 15 19))
+ (lib/filter (lib/= (lib/get-day-of-week created-at-col :iso) 1 7))
+ (lib/filter (lib/= (lib/get-year created-at-col) 2008)))]
(is (=? {:structured_output
{:type "query"
:query_id string?
- :query (mt/mbql-query products
- {:aggregation [[:metric metric-id]]
- :breakout [!week.products.created_at !day.products.created_at]
- :filter [:and
- [:> [:field %id {}] 50]
- [:= [:field %title {}] "3" "4"]
- [:!= [:field %rating {}] 3 4]
- [:= [:get-month [:field %created_at {}]] 4 5 9]
- [:!= [:get-day [:field %products.created_at {}]] 14 15 19]
- [:= [:get-day-of-week [:field %created_at {}] :iso] 1 7]
- [:= [:get-year [:field %created_at {}]] 2008]]})
:result_columns
[{:field_id (str "q" query-id "-0")
:name "CREATED_AT"
@@ -284,8 +290,8 @@
:database_type missing-value
:semantic_type "score"}]}
:conversation_id conversation-id}
- (-> response
- (update-in [:structured_output :query] mbql.normalize/normalize)))))))))
+ response))
+ (is (tools.tu/query= expected-query actual-query)))))))
(deftest query-model-test
(mt/with-premium-features #{:metabot-v3}
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/entity_details_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/entity_details_test.clj
index 47ca9be27df..072bdf48ea3 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/entity_details_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/entity_details_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer :all]
[metabase-enterprise.metabot-v3.tools.entity-details :as entity-details]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]))
@@ -157,3 +158,119 @@
(count (:fields products-related)))
(str "Products related table should have " expected-products-field-count
" fields (only its own fields, not implicitly joinable fields)"))))))))
+
+(defn- measure-definition
+ "Create an MBQL5 measure definition with a sum aggregation."
+ [table-id field-id]
+ (let [mp (mt/metadata-provider)
+ table (lib.metadata/table mp table-id)
+ query (lib/query mp table)
+ field (lib.metadata/field mp field-id)]
+ (lib/aggregate query (lib/sum field))))
+
+(defn- segment-definition
+ "Create an MBQL5 segment definition with a filter."
+ [table-id field-id value]
+ (let [mp (mt/metadata-provider)
+ table (lib.metadata/table mp table-id)
+ query (lib/query mp table)
+ field (lib.metadata/field mp field-id)]
+ (lib/filter query (lib/> field value))))
+
+(deftest get-table-details-with-measures-test
+ (testing "get-table-details returns measures when with_measures is true"
+ (let [measure-def (measure-definition (mt/id :orders) (mt/id :orders :total))]
+ (mt/with-temp [:model/Measure {measure-id :id} {:name "Total Revenue"
+ :table_id (mt/id :orders)
+ :definition measure-def}]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (testing "with_measures: false (default) does not include measures"
+ (let [result (entity-details/get-table-details {:table-id (mt/id :orders)})
+ output (:structured-output result)]
+ (is (nil? (:measures output)))))
+
+ (testing "with_measures: true includes measures for the table"
+ (let [result (entity-details/get-table-details {:table-id (mt/id :orders)
+ :with-measures? true})
+ output (:structured-output result)
+ measures (:measures output)]
+ (is (sequential? measures))
+ (is (= 1 (count measures)))
+ (let [measure (first measures)]
+ (is (= measure-id (:id measure)))
+ (is (= "Total Revenue" (:name measure)))
+ (is (map? (:definition measure)))))))))))
+
+(deftest get-table-details-with-segments-test
+ (testing "get-table-details returns segments when with_segments is true"
+ (let [segment-def (segment-definition (mt/id :orders) (mt/id :orders :total) 100)]
+ (mt/with-temp [:model/Segment {segment-id :id} {:name "High Value Orders"
+ :table_id (mt/id :orders)
+ :definition segment-def}]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (testing "with_segments: false (default) does not include segments"
+ (let [result (entity-details/get-table-details {:table-id (mt/id :orders)})
+ output (:structured-output result)]
+ (is (nil? (:segments output)))))
+
+ (testing "with_segments: true includes segments for the table"
+ (let [result (entity-details/get-table-details {:table-id (mt/id :orders)
+ :with-segments? true})
+ output (:structured-output result)
+ segments (:segments output)]
+ (is (sequential? segments))
+ (is (= 1 (count segments)))
+ (let [segment (first segments)]
+ (is (= segment-id (:id segment)))
+ (is (= "High Value Orders" (:name segment)))
+ (is (map? (:definition segment)))))))))))
+
+(deftest get-table-details-measures-scoped-to-table-test
+ (testing "get-table-details only returns measures for the requested table, not other tables"
+ (let [orders-measure-def (measure-definition (mt/id :orders) (mt/id :orders :total))
+ products-measure-def (measure-definition (mt/id :products) (mt/id :products :price))]
+ (mt/with-temp [:model/Measure {orders-measure-id :id} {:name "Orders Total"
+ :table_id (mt/id :orders)
+ :definition orders-measure-def}
+ :model/Measure {_products-measure-id :id} {:name "Products Price"
+ :table_id (mt/id :products)
+ :definition products-measure-def}]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (let [result (entity-details/get-table-details {:table-id (mt/id :orders)
+ :with-measures? true})
+ output (:structured-output result)
+ measures (:measures output)]
+ (is (= 1 (count measures)))
+ (is (= orders-measure-id (:id (first measures))))))))))
+
+(deftest get-metric-details-with-segments-test
+ (testing "get-metric-details returns segments when with_segments is true"
+ ;; Note: lib/available-segments requires the query to have a direct source-table-id.
+ ;; For metrics, this means the underlying card must be based on a table (not another card).
+ (let [mp (mt/metadata-provider)
+ metric-query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ (lib/aggregate (lib/sum (lib.metadata/field mp (mt/id :orders :total)))))
+ segment-def (segment-definition (mt/id :orders) (mt/id :orders :total) 50)]
+ (mt/with-temp [:model/Card {metric-id :id} {:dataset_query metric-query
+ :database_id (mt/id)
+ :name "Total Orders"
+ :type :metric}
+ :model/Segment {segment-id :id} {:name "Large Orders"
+ :table_id (mt/id :orders)
+ :definition segment-def}]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (testing "with_segments: false (default) does not include segments"
+ (let [result (entity-details/get-metric-details {:metric-id metric-id})
+ output (:structured-output result)]
+ (is (nil? (:segments output)))))
+
+ (testing "with_segments: true includes segments for the metric"
+ (let [result (entity-details/get-metric-details {:metric-id metric-id
+ :with-segments? true})
+ output (:structured-output result)
+ segments (:segments output)]
+ (is (sequential? segments))
+ (is (= 1 (count segments)))
+ (let [segment (first segments)]
+ (is (= segment-id (:id segment)))
+ (is (= "Large Orders" (:name segment)))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/filters_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/filters_test.clj
index 869ecd9daef..97970ed373c 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/filters_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/filters_test.clj
@@ -1,12 +1,17 @@
(ns metabase-enterprise.metabot-v3.tools.filters-test
(:require
+ [clojure.string :as str]
[clojure.test :refer :all]
[medley.core :as m]
[metabase-enterprise.metabot-v3.tools.entity-details :as metabot-v3.tools.entity-details]
[metabase-enterprise.metabot-v3.tools.filters :as metabot-v3.tools.filters]
+ [metabase-enterprise.metabot-v3.tools.test-util :as tools.tu]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.convert :as lib.convert]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.options :as lib.options]
+ [metabase.lib.test-util :as lib.tu]
[metabase.test :as mt]
[metabase.util :as u]))
@@ -42,35 +47,32 @@
(when-not <>
(throw (ex-info (str "Column " % " not found") {:column %}))))]
(testing "Trivial query works."
- (is (=? {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :orders)
- :aggregation [[:metric metric-id]]}}}}
- (metabot-v3.tools.filters/query-metric
- {:metric-id metric-id
- :filters []
- :group-by []}))))
+ (let [result (metabot-v3.tools.filters/query-metric
+ {:metric-id metric-id
+ :filters []
+ :group-by []})
+ actual-query (get-in result [:structured-output :query])
+ expected-query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ (lib/aggregate (lib.options/ensure-uuid [:metric {} metric-id])))]
+ (is (= :query (get-in result [:structured-output :type])))
+ (is (string? (get-in result [:structured-output :query-id])))
+ (is (tools.tu/query= expected-query actual-query))))
(testing "Filtering and grouping works and ignores bucketing for non-temporal columns."
(is (=? {:structured-output {:type :query,
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :orders)
- :aggregation [[:metric metric-id]]
- :breakout [[:field (mt/id :products :category)
- {:base-type :type/Text
- :source-field (mt/id :orders :product_id)}]]
- :filter
- [:and
- [:= [:field (mt/id :people :state)
- {:base-type :type/Text
- :source-field (mt/id :orders :user_id)}]
- "TX"]
- [:> [:field (mt/id :orders :discount)
- {:base-type :type/Float}]
- 3]]}}}}
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table (mt/id :orders)
+ :aggregation [[:metric {} metric-id]]
+ :breakout [[:field {:source-field (mt/id :orders :product_id)}
+ (mt/id :products :category)]]
+ :filters
+ [[:= {} [:field {:source-field (mt/id :orders :user_id)}
+ (mt/id :people :state)]
+ "TX"]
+ [:> {} [:field {} (mt/id :orders :discount)]
+ 3]]}]}}}
(metabot-v3.tools.filters/query-metric
{:metric-id metric-id
:filters [{:field-id (->field-id ["User" "State"])
@@ -85,23 +87,18 @@
(is (=? {:structured-output {:type :query,
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :orders)
- :aggregation [[:metric metric-id]]
- :filter [:and
- [:!=
- [:get-week
- [:field (mt/id :orders :created_at)
- {:base-type :type/DateTimeWithLocalTZ}]
- :iso]
- 1 2 3]
- [:=
- [:get-month [:field (mt/id :orders :created_at)
- {:base-type :type/DateTimeWithLocalTZ}]]
- 6 7 8]]
- :breakout [[:field (mt/id :orders :created_at)
- {:base-type :type/DateTimeWithLocalTZ
- :temporal-unit :week}]]}}}}
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table (mt/id :orders)
+ :aggregation [[:metric {} metric-id]]
+ :filters [[:!= {}
+ [:get-week {} [:field {} (mt/id :orders :created_at)] :iso]
+ 1 2 3]
+ [:= {}
+ [:get-month {} [:field {} (mt/id :orders :created_at)]]
+ 6 7 8]]
+ :breakout [[:field {:temporal-unit :week}
+ (mt/id :orders :created_at)]]}]}}}
(metabot-v3.tools.filters/query-metric
{:metric-id metric-id
:filters [{:field-id (->field-id "Created At")
@@ -115,29 +112,34 @@
:group-by [{:field-id (->field-id "Created At")
:field-granularity :week}]}))))
(testing "Multi-value filtering works"
- (let [state-id (->field-id ["User" "State"])
- stat-field-clause [:field (mt/id :people :state)
- {:base-type :type/Text
- :source-field (mt/id :orders :user_id)}]]
+ (let [state-id (->field-id ["User" "State"])]
(is (=? {:structured-output {:type :query,
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :orders)
- :aggregation [[:metric metric-id]]
- :filter
- [:and
- [:contains {:case-sensitive false}
- stat-field-clause "o" "e"]
- [:does-not-contain
- stat-field-clause "y" {:case-sensitive false}]
- [:starts-with {:case-sensitive false}
- stat-field-clause "A" "G"]
- [:ends-with {:case-sensitive false}
- stat-field-clause "e" "f" "i" "T" "x"]
- [:!=
- [:field (mt/id :orders :discount) {:base-type :type/Float}]
- 3 42]]}}}}
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table (mt/id :orders)
+ :aggregation [[:metric {} metric-id]]
+ :filters
+ [[:contains {:case-sensitive false}
+ [:field {:source-field (mt/id :orders :user_id)}
+ (mt/id :people :state)]
+ "o" "e"]
+ [:does-not-contain {:case-sensitive false}
+ [:field {:source-field (mt/id :orders :user_id)}
+ (mt/id :people :state)]
+ "y"]
+ [:starts-with {:case-sensitive false}
+ [:field {:source-field (mt/id :orders :user_id)}
+ (mt/id :people :state)]
+ "A" "G"]
+ [:ends-with {:case-sensitive false}
+ [:field {:source-field (mt/id :orders :user_id)}
+ (mt/id :people :state)]
+ "e" "f" "i" "T" "x"]
+ [:!= {}
+ [:field {} (mt/id :orders :discount)]
+ 3 42]]}]}}}
(metabot-v3.tools.filters/query-metric
{:metric-id metric-id
:filters [{:field-id state-id
@@ -177,7 +179,6 @@
(mt/with-current-user (mt/user->id :crowberto)
(let [model-details (-> (metabot-v3.tools.entity-details/get-table-details {:model-id model-id})
:structured-output)
- model-card-id (str "card__" model-id)
->field-id #(u/prog1 (-> model-details :fields (by-name %) :field_id)
(when-not <>
(throw (ex-info (str "Column " % " not found") {:column %}))))
@@ -185,7 +186,10 @@
(testing "Trivial query works."
(is (=? {:structured-output {:type :query,
:query-id string?
- :query (mt/mbql-query orders {:source-table model-card-id})}}
+ :query {:database (mt/id)
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card model-id}]}}}
(metabot-v3.tools.filters/query-model
{:model-id model-id
:filters []
@@ -193,11 +197,13 @@
(testing "Filtering, aggregation and grouping works and ignores bucketing for non-temporal columns."
(is (=? {:structured-output {:type :query,
:query-id string?
- :query (mt/mbql-query orders
- {:source-table model-card-id
- :aggregation [[:sum [:field "SUBTOTAL" {}]]]
- :breakout [[:field "PRODUCT_ID" {}]]
- :filter [:> [:field "DISCOUNT" {}] 3]})}}
+ :query {:database (mt/id)
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card model-id
+ :aggregation [[:sum {} [:field {} "SUBTOTAL"]]]
+ :breakout [[:field {} "PRODUCT_ID"]]
+ :filters [[:> {} [:field {} "DISCOUNT"] 3]]}]}}}
(metabot-v3.tools.filters/query-model
{:model-id model-id
:filters [{:field-id (->field-id "Discount")
@@ -211,27 +217,17 @@
(testing "Temporal bucketing works for temporal columns."
(is (=? {:structured-output {:type :query,
:query-id string?
- :query (mt/mbql-query orders
- {:source-table model-card-id
- :aggregation [[:min [:field "CREATED_AT"
- {:base-type :type/DateTimeWithLocalTZ
- :temporal-unit :hour-of-day}]]
- [:avg [:field "CREATED_AT"
- {:base-type :type/DateTimeWithLocalTZ
- :temporal-unit :hour-of-day}]]
- [:max [:field "CREATED_AT"
- {:base-type :type/DateTimeWithLocalTZ
- :temporal-unit :hour-of-day}]]]
- :filter [:and
- [:!= [:get-week [:field "CREATED_AT" {}] :iso] 1 2 3]
- [:=
- [:get-month [:field "CREATED_AT"
- {:base-type :type/DateTimeWithLocalTZ}]]
- 6 7 8]]
- :breakout [[:field "PRODUCT_ID" {}]
- [:field "CREATED_AT"
- {:base-type :type/DateTimeWithLocalTZ
- :temporal-unit :week}]]})}}
+ :query {:database (mt/id)
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card model-id
+ :aggregation [[:min {} [:field {:temporal-unit :hour-of-day} "CREATED_AT"]]
+ [:avg {} [:field {:temporal-unit :hour-of-day} "CREATED_AT"]]
+ [:max {} [:field {:temporal-unit :hour-of-day} "CREATED_AT"]]]
+ :filters [[:!= {} [:get-week {} [:field {} "CREATED_AT"] :iso] 1 2 3]
+ [:= {} [:get-month {} [:field {} "CREATED_AT"]] 6 7 8]]
+ :breakout [[:field {} "PRODUCT_ID"]
+ [:field {:temporal-unit :week} "CREATED_AT"]]}]}}}
(metabot-v3.tools.filters/query-model
{:model-id model-id
:filters [{:field-id order-created-at-field-id
@@ -287,9 +283,11 @@
(let [expected-query {:structured-output
{:type :query,
:query-id string?
- :query (mt/mbql-query orders
- {:source-table model-card-id
- :filter [:!= [:field "USER_ID" {}] 3 42]})}}
+ :query {:database (mt/id)
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card model-id
+ :filters [[:!= {} [:field {} "USER_ID"] 3 42]]}]}}}
input {:model-id model-id
:filters [{:field-id (->field-id "User ID")
:operation :not-equals
@@ -319,40 +317,34 @@
(when-not <>
(throw (ex-info (str "Column " % " not found") {:column %}))))]
(testing "Trivial query works."
- (is (=? {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :type :query
- :query {:source-table table-id}}}}
- (metabot-v3.tools.filters/filter-records
- {:data-source {:table-id table-id}
- :filters []}))))
+ (let [result (metabot-v3.tools.filters/filter-records
+ {:data-source {:table-id table-id}
+ :filters []})
+ actual-query (get-in result [:structured-output :query])
+ expected-query (lib/query mp (lib.metadata/table mp table-id))]
+ (is (= :query (get-in result [:structured-output :type])))
+ (is (string? (get-in result [:structured-output :query-id])))
+ (is (tools.tu/query= expected-query actual-query))))
(testing "Filtering works."
- (is (=? {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :orders)
- :filter
- [:and
- [:=
- [:get-day-of-week
- [:field (mt/id :orders :created_at)
- {:base-type :type/DateTimeWithLocalTZ}]
- :iso]
- 1 7]
- [:>
- [:field (mt/id :orders :discount) {:base-type :type/Float}]
- 3]]}}}}
- (metabot-v3.tools.filters/filter-records
- {:data-source {:table-id table-id}
- :filters [{:field-id (->field-id "Created At")
- :bucket :day-of-week
- :operation :equals
- :values [1 7]}
- {:field-id (->field-id "Discount")
- :operation :number-greater-than
- :value 3}]})))))
+ (let [result (metabot-v3.tools.filters/filter-records
+ {:data-source {:table-id table-id}
+ :filters [{:field-id (->field-id "Created At")
+ :bucket :day-of-week
+ :operation :equals
+ :values [1 7]}
+ {:field-id (->field-id "Discount")
+ :operation :number-greater-than
+ :value 3}]})
+ actual-query (get-in result [:structured-output :query])
+ orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ created-at-col (lib.metadata/field mp (mt/id :orders :created_at))
+ discount-col (lib.metadata/field mp (mt/id :orders :discount))
+ expected-query (-> orders-query
+ (lib/filter (lib/= (lib/get-day-of-week created-at-col :iso) 1 7))
+ (lib/filter (lib/> discount-col 3)))]
+ (is (= :query (get-in result [:structured-output :type])))
+ (is (string? (get-in result [:structured-output :query-id])))
+ (is (tools.tu/query= expected-query actual-query)))))
(testing "Missing table results in an error."
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Not found."
(metabot-v3.tools.filters/filter-records
@@ -382,8 +374,9 @@
(is (=? {:structured-output {:type :query,
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-table table-id}}}}
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card model-id}]}}}
(metabot-v3.tools.filters/filter-records
{:data-source {:table-id table-id}
:filters []}))))
@@ -391,9 +384,10 @@
(is (=? {:structured-output {:type :query,
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-table table-id
- :filter [:> [:field "DISCOUNT" {:base-type :type/Float}] 3]}}}}
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card model-id
+ :filters [[:> {} [:field {} "DISCOUNT"] 3]]}]}}}
(metabot-v3.tools.filters/filter-records
{:data-source {:table-id table-id}
:filters [{:field-id (->field-id "Discount")
@@ -428,8 +422,9 @@
(is (=? {:structured-output {:type :query,
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-table table-id}}}}
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table table-id}]}}}
(metabot-v3.tools.filters/filter-records
{:data-source {:report-id card-id}
:filters []}))))
@@ -437,13 +432,10 @@
(is (=? {:structured-output {:type :query,
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-table table-id
- :filter [:>
- [:field
- (mt/id :orders :discount)
- {:base-type :type/Float}]
- 3]}}}}
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table table-id
+ :filters [[:> {} [:field {} (mt/id :orders :discount)] 3]]}]}}}
(metabot-v3.tools.filters/filter-records
{:data-source {:report-id card-id}
:filters [{:field-id (->field-id "Discount")
@@ -472,13 +464,11 @@
expected {:structured-output {:type :query,
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-query {:source-table table-id}
- :filter [:>
- [:field
- "DISCOUNT"
- {:base-type :type/Float}]
- 3]}}}}]
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table table-id}
+ {:lib/type :mbql.stage/mbql
+ :filters [[:> {} [:field {} "DISCOUNT"] 3]]}]}}}]
(mt/with-current-user (mt/user->id :crowberto)
(testing "Filtering works."
(testing "new tool call with query and query-id"
@@ -506,8 +496,9 @@
(is (=? {:structured-output {:type :query
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-table table-id}}}}
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table table-id}]}}}
(metabot-v3.tools.filters/query-datasource
{:table-id table-id}))))
@@ -515,10 +506,11 @@
(is (=? {:structured-output {:type :query
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-table table-id
- :fields [[:field (mt/id :orders :created_at) {:base-type :type/DateTimeWithLocalTZ}]
- [:field (mt/id :orders :total) {:base-type :type/Float}]]}}}}
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table table-id
+ :fields [[:field {} (mt/id :orders :created_at)]
+ [:field {} (mt/id :orders :total)]]}]}}}
(metabot-v3.tools.filters/query-datasource
{:table-id table-id
:fields [{:field-id (->field-id "Created At")}
@@ -528,11 +520,11 @@
(is (=? {:structured-output {:type :query
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-table table-id
- :filter [:and
- [:> [:field (mt/id :orders :discount) {:base-type :type/Float}] 3]
- [:= [:field (mt/id :orders :user_id) {:base-type :type/Integer}] 10]]}}}}
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table table-id
+ :filters [[:> {} [:field {} (mt/id :orders :discount)] 3]
+ [:= {} [:field {} (mt/id :orders :user_id)] 10]]}]}}}
(metabot-v3.tools.filters/query-datasource
{:table-id table-id
:filters [{:field-id (->field-id "Discount")
@@ -546,11 +538,12 @@
(is (=? {:structured-output {:type :query
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-table table-id
- :aggregation [[:sum [:field (mt/id :orders :subtotal) {:base-type :type/Float}]]
- [:avg [:field (mt/id :orders :discount) {:base-type :type/Float}]]]
- :breakout [[:field (mt/id :orders :product_id) {:base-type :type/Integer}]]}}}}
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table table-id
+ :aggregation [[:sum {} [:field {} (mt/id :orders :subtotal)]]
+ [:avg {} [:field {} (mt/id :orders :discount)]]]
+ :breakout [[:field {} (mt/id :orders :product_id)]]}]}}}
(metabot-v3.tools.filters/query-datasource
{:table-id table-id
:aggregations [{:field-id (->field-id "Subtotal")
@@ -563,11 +556,12 @@
(is (=? {:structured-output {:type :query
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-table table-id
- :order-by [[:asc [:field (mt/id :orders :created_at) {:base-type :type/DateTimeWithLocalTZ}]]
- [:desc [:field (mt/id :orders :total) {:base-type :type/Float}]]]
- :limit 100}}}}
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table table-id
+ :order-by [[:asc {} [:field {} (mt/id :orders :created_at)]]
+ [:desc {} [:field {} (mt/id :orders :total)]]]
+ :limit 100}]}}}
(metabot-v3.tools.filters/query-datasource
{:table-id table-id
:order-by [{:field {:field-id (->field-id "Created At")}
@@ -580,12 +574,11 @@
(is (=? {:structured-output {:type :query
:query-id string?
:query {:database (mt/id)
- :type :query
- :query {:source-table table-id
- :aggregation [[:count]]
- :breakout [[:field (mt/id :orders :created_at)
- {:base-type :type/DateTimeWithLocalTZ
- :temporal-unit :month}]]}}}}
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-table table-id
+ :aggregation [[:count {}]]
+ :breakout [[:field {:temporal-unit :month} (mt/id :orders :created_at)]]}]}}}
(metabot-v3.tools.filters/query-datasource
{:table-id table-id
:aggregations [{:field-id (->field-id "Product ID")
@@ -602,24 +595,28 @@
(mt/with-current-user (mt/user->id :crowberto)
(let [model-details (-> (metabot-v3.tools.entity-details/get-table-details {:model-id model-id})
:structured-output)
- model-card-id (str "card__" model-id)
->field-id #(u/prog1 (-> model-details :fields (by-name %) :field_id)
(when-not <>
(throw (ex-info (str "Column " % " not found") {:column %}))))]
(testing "Basic model query works"
(is (=? {:structured-output {:type :query
:query-id string?
- :query (mt/mbql-query orders {:source-table model-card-id})}}
+ :query {:database (mt/id)
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card model-id}]}}}
(metabot-v3.tools.filters/query-datasource
{:model-id model-id}))))
(testing "Model query with fields selection"
(is (=? {:structured-output {:type :query
:query-id string?
- :query (mt/mbql-query orders
- {:source-table model-card-id
- :fields [[:field "CREATED_AT" {:base-type :type/DateTimeWithLocalTZ}]
- [:field "TOTAL" {:base-type :type/Float}]]})}}
+ :query {:database (mt/id)
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card model-id
+ :fields [[:field {} "CREATED_AT"]
+ [:field {} "TOTAL"]]}]}}}
(metabot-v3.tools.filters/query-datasource
{:model-id model-id
:fields [{:field-id (->field-id "Created At")}
@@ -628,11 +625,12 @@
(testing "Model query with filters"
(is (=? {:structured-output {:type :query
:query-id string?
- :query (mt/mbql-query orders
- {:source-table model-card-id
- :filter [:and
- [:> [:field "DISCOUNT" {:base-type :type/Float}] 5]
- [:< [:field "SUBTOTAL" {:base-type :type/Float}] 100]]})}}
+ :query {:database (mt/id)
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card model-id
+ :filters [[:> {} [:field {} "DISCOUNT"] 5]
+ [:< {} [:field {} "SUBTOTAL"] 100]]}]}}}
(metabot-v3.tools.filters/query-datasource
{:model-id model-id
:filters [{:field-id (->field-id "Discount")
@@ -645,11 +643,13 @@
(testing "Model query with aggregations and grouping"
(is (=? {:structured-output {:type :query
:query-id string?
- :query (mt/mbql-query orders
- {:source-table model-card-id
- :aggregation [[:count]
- [:max [:field "TOTAL" {:base-type :type/Float}]]]
- :breakout [[:field "USER_ID" {:base-type :type/Integer}]]})}}
+ :query {:database (mt/id)
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card model-id
+ :aggregation [[:count {}]
+ [:max {} [:field {} "TOTAL"]]]
+ :breakout [[:field {} "USER_ID"]]}]}}}
(metabot-v3.tools.filters/query-datasource
{:model-id model-id
:aggregations [{:field-id (->field-id "Product ID")
@@ -661,10 +661,12 @@
(testing "Model query with order by and limit"
(is (=? {:structured-output {:type :query
:query-id string?
- :query (mt/mbql-query orders
- {:source-table model-card-id
- :order-by [[:desc [:field "CREATED_AT" {:base-type :type/DateTimeWithLocalTZ}]]]
- :limit 50})}}
+ :query {:database (mt/id)
+ :lib/type :mbql/query
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card model-id
+ :order-by [[:desc {} [:field {} "CREATED_AT"]]]
+ :limit 50}]}}}
(metabot-v3.tools.filters/query-datasource
{:model-id model-id
:order-by [{:field {:field-id (->field-id "Created At")}
@@ -717,3 +719,170 @@
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."
(metabot-v3.tools.filters/query-datasource
{:model-id model-id}))))))
+
+(defn- mbql-measure-definition
+ "Create an MBQL5 measure definition with a sum aggregation."
+ [table-id field-id]
+ (let [mp (mt/metadata-provider)
+ table (lib.metadata/table mp table-id)
+ query (lib/query mp table)
+ field (lib.metadata/field mp field-id)]
+ (lib/aggregate query (lib/sum field))))
+
+(defn- mbql-segment-definition
+ "Create an MBQL5 segment definition with a filter."
+ [table-id field-id value]
+ (let [mp (mt/metadata-provider)
+ table (lib.metadata/table mp table-id)
+ query (lib/query mp table)
+ field (lib.metadata/field mp field-id)]
+ (lib/filter query (lib/> field value))))
+
+(defn- add-mock-segment
+ "Add a mock segment to a metadata provider."
+ [base-mp segment-id table-id definition]
+ (lib.tu/mock-metadata-provider
+ base-mp
+ {:segments [{:lib/type :metadata/segment
+ :id segment-id
+ :name (str "Mock Segment " segment-id)
+ :table-id table-id
+ :archived false
+ :definition definition}]}))
+
+(defn- add-mock-measure
+ "Add a mock measure to a metadata provider."
+ [base-mp measure-id table-id definition]
+ (lib.tu/mock-metadata-provider
+ base-mp
+ {:measures [{:lib/type :metadata/measure
+ :id measure-id
+ :name (str "Mock Measure " measure-id)
+ :table-id table-id
+ :archived false
+ :definition definition}]}))
+
+(deftest query-metric-with-segment-filter-test
+ (let [mp (mt/metadata-provider)
+ created-at-meta (lib.metadata/field mp (mt/id :orders :created_at))
+ metric-query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ (lib/aggregate (lib/sum (lib.metadata/field mp (mt/id :orders :total))))
+ (lib/breakout (lib/with-temporal-bucket created-at-meta :month)))
+ legacy-metric-query (lib.convert/->legacy-MBQL metric-query)
+ segment-def (mbql-segment-definition (mt/id :orders) (mt/id :orders :total) 50)
+ mock-segment-id 1
+ mock-mp (add-mock-segment mp mock-segment-id (mt/id :orders) segment-def)]
+ (mt/with-temp [:model/Card {metric-id :id} {:dataset_query legacy-metric-query
+ :database_id (mt/id)
+ :name "Total Orders"
+ :type :metric}]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (testing "query-metric with segment filter applies the segment"
+ (with-redefs [lib-be/application-database-metadata-provider (constantly mock-mp)]
+ (let [result (metabot-v3.tools.filters/query-metric
+ {:metric-id metric-id
+ :filters [{:segment-id mock-segment-id}]
+ :group-by []})
+ query (get-in result [:structured-output :query])
+ filters (get-in query [:stages 0 :filters])]
+ (is (some? (:structured-output result)))
+ (is (= :query (get-in result [:structured-output :type])))
+ (is (=? [[:segment {} mock-segment-id]] filters)))))))))
+
+(deftest query-datasource-with-measure-aggregation-test
+ (let [mp (mt/metadata-provider)
+ measure-def (mbql-measure-definition (mt/id :orders) (mt/id :orders :total))
+ mock-measure-id 1
+ mock-mp (add-mock-measure mp mock-measure-id (mt/id :orders) measure-def)]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (with-redefs [lib-be/application-database-metadata-provider (constantly mock-mp)]
+ (testing "query-datasource with measure aggregation"
+ (let [result (metabot-v3.tools.filters/query-datasource
+ {:table-id (mt/id :orders)
+ :aggregations [{:measure-id mock-measure-id}]})
+ query (get-in result [:structured-output :query])
+ aggregations (get-in query [:stages 0 :aggregation])]
+ (is (some? (:structured-output result)))
+ (is (= :query (get-in result [:structured-output :type])))
+ ;; Check that a measure aggregation is present
+ (is (some #(and (vector? %) (= :measure (first %))) aggregations))))
+
+ (testing "query-datasource with measure aggregation and sort order"
+ (let [result (metabot-v3.tools.filters/query-datasource
+ {:table-id (mt/id :orders)
+ :aggregations [{:measure-id mock-measure-id
+ :sort-order :desc}]})
+ query (get-in result [:structured-output :query])
+ order-by (get-in query [:stages 0 :order-by])]
+ (is (some? order-by))
+ (is (some #(= :desc (first %)) order-by))))))))
+
+(deftest query-datasource-with-segment-filter-test
+ (let [mp (mt/metadata-provider)
+ segment-def (mbql-segment-definition (mt/id :orders) (mt/id :orders :total) 100)
+ mock-segment-id 1
+ mock-mp (add-mock-segment mp mock-segment-id (mt/id :orders) segment-def)]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (with-redefs [lib-be/application-database-metadata-provider (constantly mock-mp)]
+ (testing "query-datasource with segment filter"
+ (let [result (metabot-v3.tools.filters/query-datasource
+ {:table-id (mt/id :orders)
+ :filters [{:segment-id mock-segment-id}]})
+ query (get-in result [:structured-output :query])
+ filters (get-in query [:stages 0 :filters])]
+ (is (some? (:structured-output result)))
+ (is (= :query (get-in result [:structured-output :type])))
+ ;; Check that a segment filter is present
+ (is (some #(and (vector? %) (= :segment (first %))) filters))))))))
+
+(deftest query-datasource-with-measures-and-segments-test
+ (let [mp (mt/metadata-provider)
+ measure-def (mbql-measure-definition (mt/id :orders) (mt/id :orders :total))
+ segment-def (mbql-segment-definition (mt/id :orders) (mt/id :orders :discount) 0)
+ mock-measure-id 1
+ mock-segment-id 1
+ mock-mp (-> mp
+ (add-mock-measure mock-measure-id (mt/id :orders) measure-def)
+ (add-mock-segment mock-segment-id (mt/id :orders) segment-def))]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (with-redefs [lib-be/application-database-metadata-provider (constantly mock-mp)]
+ (testing "query-datasource with both measure aggregation and segment filter"
+ (let [result (metabot-v3.tools.filters/query-datasource
+ {:table-id (mt/id :orders)
+ :aggregations [{:measure-id mock-measure-id}]
+ :filters [{:segment-id mock-segment-id}]})
+ query (get-in result [:structured-output :query])
+ aggregations (get-in query [:stages 0 :aggregation])
+ filters (get-in query [:stages 0 :filters])]
+ (is (some? (:structured-output result)))
+ ;; Check that both measure and segment are present
+ (is (some #(and (vector? %) (= :measure (first %))) aggregations))
+ (is (some #(and (vector? %) (= :segment (first %))) filters))))))))
+
+(deftest segment-not-found-test
+ (let [mp (mt/metadata-provider)
+ non-existent-segment-id 99999]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (with-redefs [lib-be/application-database-metadata-provider (constantly mp)]
+ (testing "query-datasource with non-existent segment returns error"
+ (let [result (metabot-v3.tools.filters/query-datasource
+ {:table-id (mt/id :orders)
+ :filters [{:segment-id non-existent-segment-id}]})]
+ (is (nil? (:structured-output result)))
+ (is (string? (:output result)))
+ (is (str/includes? (:output result) "Segment"))
+ (is (str/includes? (:output result) "not found"))))))))
+
+(deftest measure-not-found-test
+ (let [mp (mt/metadata-provider)
+ non-existent-measure-id 99999]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (with-redefs [lib-be/application-database-metadata-provider (constantly mp)]
+ (testing "query-datasource with non-existent measure returns error"
+ (let [result (metabot-v3.tools.filters/query-datasource
+ {:table-id (mt/id :orders)
+ :aggregations [{:measure-id non-existent-measure-id}]})]
+ (is (nil? (:structured-output result)))
+ (is (string? (:output result)))
+ (is (str/includes? (:output result) "Measure"))
+ (is (str/includes? (:output result) "not found"))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/test_util.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/test_util.clj
new file mode 100644
index 00000000000..43fa71ec3ad
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/test_util.clj
@@ -0,0 +1,9 @@
+(ns metabase-enterprise.metabot-v3.tools.test-util
+ (:require
+ [metabase.lib.core :as lib]))
+
+(defn query=
+ "Compare two queries for equality, ignoring lib/uuids."
+ [expected actual]
+ (= (lib/remove-lib-uuids expected)
+ (lib/remove-lib-uuids actual)))