The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Support measures and segments for Metabot (#67368)
Thomas Schmidt5ab27ac0fbc4
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.cljindex eae3eb37869..c6f792b7b1f 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj@@ -163,8 +163,16 @@[:values [:sequential [:or :int :double]]]][:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])+(mr/def ::segment-filter+ "Filter using a pre-defined segment."+ [:and+ [:map+ [:segment_id :int]]+ [:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])+(mr/def ::filter[:or+ ::segment-filter::existence-filter::temporal-extraction-filter ::disjunctive-temporal-extraction-filter::temporal-filter ::disjunctive-temporal-filter@@ -180,7 +188,8 @@"minute", "hour" "day" "week" "month" "quarter" "year" "day-of-week"]]]][:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])-(mr/def ::aggregation+(mr/def ::field-aggregation+ "Aggregation using a field and function."[:and[:map[:field_id :string]@@ -190,6 +199,18 @@"avg" "count" "count-distinct" "max" "min" "sum"]]][:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])+(mr/def ::measure-aggregation+ "Aggregation using a pre-defined measure."+ [:and+ [:map+ [:measure_id :int]+ [:sort_order {:optional true} [:maybe [:enum {:encode/tool-api-request keyword} "asc" "desc"]]]]+ [:map {:encode/tool-api-request #(update-keys % metabot-v3.u/safe->kebab-case-en)}]])++(mr/def ::aggregation+ "Aggregation - either field-based or measure-based."+ [:or ::field-aggregation ::measure-aggregation])+(mr/def ::field[:and[:map@@ -282,8 +303,25 @@::basic-metric[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}[:queryable_dimensions {:optional true} ::columns]+ [:segments {:optional true} [:sequential ::segment]][:verified {:optional true} :boolean]]])+(mr/def ::measure+ [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}+ [:id :int]+ [:name :string]+ [:display_name {:optional true} [:maybe :string]]+ [:description {:optional true} [:maybe :string]]+ [:definition {:optional true} [:maybe :map]]])++(mr/def ::segment+ [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}+ [:id :int]+ [:name :string]+ [:display_name {:optional true} [:maybe :string]]+ [:description {:optional true} [:maybe :string]]+ [:definition {:optional true} [:maybe :map]]])+(mr/def ::table-result[:map[:id :int]@@ -297,7 +335,9 @@[:related_tables {:optional true} [:sequential [:ref ::table-result]]][:related_by {:optional true} [:maybe :string]][:description {:optional true} [:maybe :string]]- [:metrics {:optional true} [:sequential ::basic-metric]]])+ [:metrics {:optional true} [:sequential ::basic-metric]]+ [:measures {:optional true} [:sequential ::measure]]+ [:segments {:optional true} [:sequential ::segment]]])(mr/def ::basic-transform[:map@@ -590,12 +630,14 @@[:metric_id :int][:with_default_temporal_breakout {:optional true, :default true} :boolean][:with_field_values {:optional true, :default true} :boolean]- [:with_queryable_dimensions {:optional true, :default true} :boolean]]+ [:with_queryable_dimensions {:optional true, :default true} :boolean]+ [:with_segments {:optional true, :default false} :boolean]][:map {:encode/tool-api-request#(set/rename-keys % {:metric_id :metric-id:with_default_temporal_breakout :with-default-temporal-breakout?:with_field_values :with-field-values?- :with_queryable_dimensions :with-queryable-dimensions?})}]])+ :with_queryable_dimensions :with-queryable-dimensions?+ :with_segments :with-segments?})}]])(mr/def ::get-metric-details-result[:or@@ -694,7 +736,9 @@[:with_field_values {:optional true, :default true} :boolean][:with_related_tables {:optional true, :default true} :boolean][:with_metrics {:optional true, :default true} :boolean]- [:with_metric_default_temporal_breakout {:optional true, :default true} :boolean]]+ [:with_metric_default_temporal_breakout {:optional true, :default true} :boolean]+ [:with_measures {:optional true, :default false} :boolean]+ [:with_segments {:optional true, :default false} :boolean]][:fn {:error/message "Exactly one of model_id and table_id required"}#(= (count (select-keys % [:model_id :table_id])) 1)][:map {:encode/tool-api-request@@ -704,7 +748,9 @@:with_field_values :with-field-values?:with_related_tables :with-related-tables?:with_metrics :with-metrics?- :with_metric_default_temporal_breakout :with-default-temporal-breakout?})}]])+ :with_metric_default_temporal_breakout :with-default-temporal-breakout?+ :with_measures :with-measures?+ :with_segments :with-segments?})}]])(mr/def ::get-table-details-result[:ordiff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/entity_details.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/entity_details.cljindex ebc214bbc90..1b3bb01f7ce 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/entity_details.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/entity_details.clj@@ -18,6 +18,11 @@(set! *warn-on-reflection* true)+(defn- convert-measure-or-segment+ "Convert a measure or segment metadata object to the format expected by the API."+ [metadata]+ (select-keys metadata [:id :name :display-name :description :definition]))+(defn verified-review?"Return true if the most recent ModerationReview for the given item id/type is verified."[id item-type]@@ -94,12 +99,14 @@([id options](when-let [card (metabot-v3.tools.u/get-card id)](metric-details card (lib-be/application-database-metadata-provider (:database_id card)) options)))- ([card metadata-provider {:keys [field-values-fn with-default-temporal-breakout? with-queryable-dimensions?]+ ([card metadata-provider {:keys [field-values-fn with-default-temporal-breakout? with-queryable-dimensions?+ with-segments?]:or {field-values-fn add-field-valueswith-default-temporal-breakout? true- with-queryable-dimensions? true}}]+ with-queryable-dimensions? true+ with-segments? false}}](let [id (:id card)- query-needed? (or with-default-temporal-breakout? with-queryable-dimensions?)+ query-needed? (or with-default-temporal-breakout? with-queryable-dimensions? with-segments?)metric-query (when query-needed?(lib/query metadata-provider (lib.metadata/card metadata-provider id)))breakouts (when query-needed?@@ -136,7 +143,12 @@(map #(metabot-v3.tools.u/->result-columnmetric-query % (col-index %) field-id-prefix)))(->> (lib/filterable-columns base-query)- field-values-fn)))))))+ field-values-fn)))++ with-segments?+ (assoc :segments (if-let [segments (lib/available-segments metric-query)]+ (mapv convert-measure-or-segment segments)+ []))))))(defn- convert-metric([db-metric metadata-provider]@@ -150,16 +162,19 @@(defn- table-details([id] (table-details id nil))- ([id {:keys [metadata-provider field-values-fn with-fields? with-related-tables? with-metrics?]+ ([id {:keys [metadata-provider field-values-fn with-fields? with-related-tables? with-metrics?+ with-measures? with-segments?]:or {field-values-fn add-field-valueswith-fields? truewith-related-tables? true- with-metrics? true}+ with-metrics? true+ with-measures? false+ with-segments? false}:as options}](when-let [base (if metadata-provider(lib.metadata/table metadata-provider id)(metabot-v3.tools.u/get-table id :db_id :description :name :schema))]- (let [query-needed? (or with-fields? with-related-tables? with-metrics?)+ (let [query-needed? (or with-fields? with-related-tables? with-metrics? with-measures? with-segments?)db-id (if metadata-provider (:db-id base) (:db_id base))db-engine (:engine (if metadata-provider(lib.metadata/database metadata-provider)@@ -191,7 +206,13 @@:related_tables related-tables:metrics (when with-metrics?(not-empty (mapv #(convert-metric % mp options)- (lib/available-metrics table-query))))))))))+ (lib/available-metrics table-query))))+ :measures (when with-measures?+ (not-empty (mapv convert-measure-or-segment+ (lib/available-measures table-query))))+ :segments (when with-segments?+ (not-empty (mapv convert-measure-or-segment+ (lib/available-segments table-query))))))))))(defn related-tables"Constructs a list of tables, optionally including their fields, that are related to the given query via foreign key.@@ -236,18 +257,21 @@([id options](when-let [card (metabot-v3.tools.u/get-card id)](card-details card (lib-be/application-database-metadata-provider (:database_id card)) options)))- ([base metadata-provider {:keys [field-values-fn with-fields? with-related-tables? with-metrics?]+ ([base metadata-provider {:keys [field-values-fn with-fields? with-related-tables? with-metrics?+ with-measures? with-segments?]:or {field-values-fn add-field-valueswith-fields? truewith-related-tables? true- with-metrics? true}+ with-metrics? true+ with-measures? false+ with-segments? false}:as options}](let [id (:id base)database-id (:database_id base)database-engine (:engine (lib.metadata/database metadata-provider))card-metadata (lib.metadata/card metadata-provider id)dataset-query (get card-metadata :dataset-query)- query-needed? (or with-fields? with-related-tables? with-metrics?)+ query-needed? (or with-fields? with-related-tables? with-metrics? with-measures? with-segments?);; pivot questions have strange result-columns so we work with the dataset-querycard-type (:type base)card-query (when query-needed?@@ -276,7 +300,13 @@:related_tables related-tables:metrics (when with-metrics?(not-empty (mapv #(convert-metric % metadata-provider options)- (lib/available-metrics card-query)))))))))+ (lib/available-metrics card-query))))+ :measures (when with-measures?+ (not-empty (mapv convert-measure-or-segment+ (lib/available-measures card-query))))+ :segments (when with-segments?+ (not-empty (mapv convert-measure-or-segment+ (lib/available-segments card-query)))))))))(defn cards-details"Get the details of metrics or models as specified by `card-type` and `cards`diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/filters.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/filters.cljindex 5ab3dd3a8a5..bd8f07ff93b 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/filters.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/filters.clj@@ -8,7 +8,8 @@[metabase.lib.options :as lib.options][metabase.lib.types.isa :as lib.types.isa][metabase.lib.util :as lib.util]- [metabase.util :as u]))+ [metabase.util :as u]+ [metabase.util.i18n :refer [tru]]))(defn- apply-filter-bucket[column bucket]@@ -76,66 +77,74 @@(defn- add-filter[query llm-filter]- (let [{:keys [operation value values]} llm-filter- expr (filter-bucketed-column llm-filter)- with-values-or-value (fn with-values-or-value- ([f]- (with-values-or-value f expr))- ([f expr]- (if values- (apply f expr values)- (f expr value))))- string-match (fn [match-fn]- (-> (with-values-or-value match-fn)- (lib.options/update-options assoc :case-sensitive false)))- filter- (case operation- :is-null (lib/is-null expr)- :is-not-null (lib/not-null expr)- :string-is-empty (lib/is-empty expr)- :string-is-not-empty (lib/not-empty expr)- :is-true (lib/= expr true)- :is-false (lib/= expr false)- :equals (with-values-or-value lib/=)- :not-equals (with-values-or-value lib/!=)- :greater-than (lib/> expr value)- :greater-than-or-equal (lib/>= expr value)- :less-than (lib/< expr value)- :less-than-or-equal (lib/<= expr value)- :year-equals (with-values-or-value lib/= (lib/get-year expr))- :year-not-equals (with-values-or-value lib/!= (lib/get-year expr))- :quarter-equals (with-values-or-value lib/= (lib/get-quarter expr))- :quarter-not-equals (with-values-or-value lib/!= (lib/get-quarter expr))- :month-equals (with-values-or-value lib/= (lib/get-month expr))- :month-not-equals (with-values-or-value lib/!= (lib/get-month expr))- :day-of-week-equals (with-values-or-value lib/= (lib/get-day-of-week expr :iso))- :day-of-week-not-equals (with-values-or-value lib/!= (lib/get-day-of-week expr :iso))- :hour-equals (with-values-or-value lib/= (lib/get-hour expr))- :hour-not-equals (with-values-or-value lib/!= (lib/get-hour expr))- :minute-equals (with-values-or-value lib/= (lib/get-minute expr))- :minute-not-equals (with-values-or-value lib/!= (lib/get-minute expr))- :second-equals (with-values-or-value lib/= (lib/get-second expr))- :second-not-equals (with-values-or-value lib/!= (lib/get-second expr))- :date-equals (with-values-or-value lib/=)- :date-not-equals (with-values-or-value lib/!=)- :date-before (lib/< expr value)- :date-on-or-before (lib/<= expr value)- :date-after (lib/> expr value)- :date-on-or-after (lib/>= expr value)- :string-equals (with-values-or-value lib/=)- :string-not-equals (with-values-or-value lib/!=)- :string-contains (string-match lib/contains)- :string-not-contains (string-match lib/does-not-contain)- :string-starts-with (string-match lib/starts-with)- :string-ends-with (string-match lib/ends-with)- :number-equals (with-values-or-value lib/=)- :number-not-equals (with-values-or-value lib/!=)- :number-greater-than (lib/> expr value)- :number-greater-than-or-equal (lib/>= expr value)- :number-less-than (lib/< expr value)- :number-less-than-or-equal (lib/<= expr value)- (throw (ex-info (str "unknown filter operation " operation) {:agent-error? true})))]- (lib/filter query filter)))+ (if-let [segment-id (:segment-id llm-filter)]+ ;; Segment-based filter+ (if-let [segment (lib.metadata/segment query segment-id)]+ (lib/filter query segment)+ (throw (ex-info (tru "Segment with id {0} not found" segment-id)+ {:agent-error? true+ :segment-id segment-id})))+ ;; Standard field-based filter logic+ (let [{:keys [operation value values]} llm-filter+ expr (filter-bucketed-column llm-filter)+ with-values-or-value (fn with-values-or-value+ ([f]+ (with-values-or-value f expr))+ ([f expr]+ (if values+ (apply f expr values)+ (f expr value))))+ string-match (fn [match-fn]+ (-> (with-values-or-value match-fn)+ (lib.options/update-options assoc :case-sensitive false)))+ filter+ (case operation+ :is-null (lib/is-null expr)+ :is-not-null (lib/not-null expr)+ :string-is-empty (lib/is-empty expr)+ :string-is-not-empty (lib/not-empty expr)+ :is-true (lib/= expr true)+ :is-false (lib/= expr false)+ :equals (with-values-or-value lib/=)+ :not-equals (with-values-or-value lib/!=)+ :greater-than (lib/> expr value)+ :greater-than-or-equal (lib/>= expr value)+ :less-than (lib/< expr value)+ :less-than-or-equal (lib/<= expr value)+ :year-equals (with-values-or-value lib/= (lib/get-year expr))+ :year-not-equals (with-values-or-value lib/!= (lib/get-year expr))+ :quarter-equals (with-values-or-value lib/= (lib/get-quarter expr))+ :quarter-not-equals (with-values-or-value lib/!= (lib/get-quarter expr))+ :month-equals (with-values-or-value lib/= (lib/get-month expr))+ :month-not-equals (with-values-or-value lib/!= (lib/get-month expr))+ :day-of-week-equals (with-values-or-value lib/= (lib/get-day-of-week expr :iso))+ :day-of-week-not-equals (with-values-or-value lib/!= (lib/get-day-of-week expr :iso))+ :hour-equals (with-values-or-value lib/= (lib/get-hour expr))+ :hour-not-equals (with-values-or-value lib/!= (lib/get-hour expr))+ :minute-equals (with-values-or-value lib/= (lib/get-minute expr))+ :minute-not-equals (with-values-or-value lib/!= (lib/get-minute expr))+ :second-equals (with-values-or-value lib/= (lib/get-second expr))+ :second-not-equals (with-values-or-value lib/!= (lib/get-second expr))+ :date-equals (with-values-or-value lib/=)+ :date-not-equals (with-values-or-value lib/!=)+ :date-before (lib/< expr value)+ :date-on-or-before (lib/<= expr value)+ :date-after (lib/> expr value)+ :date-on-or-after (lib/>= expr value)+ :string-equals (with-values-or-value lib/=)+ :string-not-equals (with-values-or-value lib/!=)+ :string-contains (string-match lib/contains)+ :string-not-contains (string-match lib/does-not-contain)+ :string-starts-with (string-match lib/starts-with)+ :string-ends-with (string-match lib/ends-with)+ :number-equals (with-values-or-value lib/=)+ :number-not-equals (with-values-or-value lib/!=)+ :number-greater-than (lib/> expr value)+ :number-greater-than-or-equal (lib/>= expr value)+ :number-less-than (lib/< expr value)+ :number-less-than-or-equal (lib/<= expr value)+ (throw (ex-info (str "unknown filter operation " operation) {:agent-error? true})))]+ (lib/filter query filter))))(defn- add-breakout[query {:keys [column field-granularity]}]@@ -146,17 +155,18 @@(lib/breakout query expr)))(defn- query-metric*- [{:keys [metric-id filters group-by]}]+ [{:keys [metric-id filters group-by] :as _arguments}](let [card (metabot-v3.tools.u/get-card metric-id)mp (lib-be/application-database-metadata-provider (:database_id card))base-query (->> (lib/query mp (lib.metadata/card mp metric-id))lib/remove-all-breakouts)field-id-prefix (metabot-v3.tools.u/card-field-id-prefix metric-id)visible-cols (lib/visible-columns base-query)+ resolve-visible-column #(metabot-v3.tools.u/resolve-column % field-id-prefix visible-cols)+ ;; Separate segment filters from field filters before column resolution+ resolved-filters (map #(if (:segment-id %) % (resolve-visible-column %)) filters)query (as-> base-query $q- (reduce add-filter- $q- (map #(metabot-v3.tools.u/resolve-column % field-id-prefix visible-cols) filters))+ (reduce add-filter $q resolved-filters)(reduce add-breakout$q(map #(metabot-v3.tools.u/resolve-column % field-id-prefix visible-cols) group-by)))@@ -165,8 +175,7 @@returned-cols (lib/returned-columns query)]{:type :query:query-id query-id- ;; existing usage, don't do this going forward -- use Lib instead and persist MBQL 5 to the app DB- :query #_{:clj-kondo/ignore [:discouraged-var]} (lib/->legacy-MBQL query)+ :query query:result-columns (into [](map-indexed #(metabot-v3.tools.u/->result-column query %2 %1 query-field-id-prefix))returned-cols)}))@@ -183,23 +192,37 @@{:output (str "No metric found with metric_id " metric-id)}(metabot-v3.tools.u/handle-agent-error e)))))+(defn- apply-aggregation-sort-order+ "If sort-order is specified, add an order-by clause for the last aggregation in the query."+ [query sort-order]+ (if sort-order+ (let [query-aggregations (lib/aggregations query)+ last-aggregation-idx (dec (count query-aggregations))]+ (lib/order-by query (lib/aggregation-ref query last-aggregation-idx) sort-order))+ query))+(defn- add-aggregation[query aggregation]- (let [expr (bucketed-column aggregation)- sort-order (:sort-order aggregation)- agg-expr (case (:function aggregation)- :count (lib/count)- :count-distinct (lib/distinct expr)- :sum (lib/sum expr)- :min (lib/min expr)- :max (lib/max expr)- :avg (lib/avg expr))- query-with-aggregation (lib/aggregate query agg-expr)]- (if sort-order- (let [query-aggregations (lib/aggregations query-with-aggregation)- last-aggregation-idx (dec (count query-aggregations))]- (lib/order-by query-with-aggregation (lib/aggregation-ref query-with-aggregation last-aggregation-idx) sort-order))- query-with-aggregation)))+ (let [sort-order (:sort-order aggregation)+ query-with-aggregation+ (if-let [measure-id (:measure-id aggregation)]+ ;; Measure-based aggregation+ (if-let [measure (lib.metadata/measure query measure-id)]+ (lib/aggregate query measure)+ (throw (ex-info (tru "Measure with id {0} not found" measure-id)+ {:agent-error? true+ :measure-id measure-id})))+ ;; Field-based aggregation+ (let [expr (bucketed-column aggregation)+ agg-expr (case (:function aggregation)+ :count (lib/count)+ :count-distinct (lib/distinct expr)+ :sum (lib/sum expr)+ :min (lib/min expr)+ :max (lib/max expr)+ :avg (lib/avg expr))]+ (lib/aggregate query agg-expr)))]+ (apply-aggregation-sort-order query-with-aggregation sort-order)))(defn- expression?[expr-or-column]@@ -219,16 +242,18 @@(lib/order-by query (:column field) direction))(defn- add-limit [query limit]- (cond-> query limit (lib/limit limit)))+ (if limit+ (lib/limit query limit)+ query))(defn- query-model*- [{:keys [model-id fields filters aggregations group-by order-by limit]}]+ [{:keys [model-id fields filters aggregations group-by order-by limit] :as _arguments}](let [card (metabot-v3.tools.u/get-card model-id)mp (lib-be/application-database-metadata-provider (:database_id card))base-query (lib/query mp (lib.metadata/card mp model-id))field-id-prefix (metabot-v3.tools.u/card-field-id-prefix model-id)visible-cols (lib/visible-columns base-query)- resolve-visible-column #(metabot-v3.tools.u/resolve-column % field-id-prefix visible-cols)+ resolve-visible-column #(metabot-v3.tools.u/resolve-column % field-id-prefix visible-cols)resolve-order-by-column (fn [{:keys [field direction]}] {:field (resolve-visible-column field) :direction direction})projection (map (comp (juxt filter-bucketed-column (fn [{:keys [column bucket]}](let [column (cond-> column@@ -236,24 +261,26 @@(lib/display-name base-query -1 column :long))))resolve-visible-column)fields)- query (as-> base-query query- (reduce (fn [query [expr-or-column expr-name]]- (lib/expression query expr-name expr-or-column))- query- (filter (comp expression? first) projection))- (add-fields query projection)- (reduce add-filter query (map resolve-visible-column filters))- (reduce add-aggregation query (map resolve-visible-column aggregations))- (reduce add-breakout query (map resolve-visible-column group-by))- (reduce add-order-by query (map resolve-order-by-column order-by))- (add-limit query limit))+ ;; Measures and segments don't require column resolution+ resolved-aggregations (map #(if (:measure-id %) % (resolve-visible-column %)) aggregations)+ resolved-filters (map #(if (:segment-id %) % (resolve-visible-column %)) filters)+ reduce-query (fn [query f coll] (reduce f query coll))+ query (-> base-query+ (reduce-query (fn [query [expr-or-column expr-name]]+ (lib/expression query expr-name expr-or-column))+ (filter (comp expression? first) projection))+ (add-fields projection)+ (reduce-query add-filter resolved-filters)+ (reduce-query add-aggregation resolved-aggregations)+ (reduce-query add-breakout (map resolve-visible-column group-by))+ (reduce-query add-order-by (map resolve-order-by-column order-by))+ (add-limit limit))query-id (u/generate-nano-id)query-field-id-prefix (metabot-v3.tools.u/query-field-id-prefix query-id)returned-cols (lib/returned-columns query)]{:type :query:query-id query-id- ;; existing usage, don't do this going forward -- use Lib instead and persist MBQL 5 to the app DB- :query #_{:clj-kondo/ignore [:discouraged-var]} (lib/->legacy-MBQL query)+ :query query:result-columns (into [](map-indexed #(metabot-v3.tools.u/->result-column query %2 %1 query-field-id-prefix))returned-cols)}))@@ -288,7 +315,7 @@[{:keys [fields filters aggregations group-by order-by limit] :as arguments}](let [[filter-field-id-prefix base-query] (resolve-datasource arguments)visible-cols (lib/visible-columns base-query)- resolve-visible-column #(metabot-v3.tools.u/resolve-column % filter-field-id-prefix visible-cols)+ resolve-visible-column #(metabot-v3.tools.u/resolve-column % filter-field-id-prefix visible-cols)resolve-order-by-column (fn [{:keys [field direction]}] {:field (resolve-visible-column field) :direction direction})projection (map (comp (juxt filter-bucketed-column (fn [{:keys [column bucket]}](let [column (cond-> column@@ -296,24 +323,26 @@(lib/display-name base-query -1 column :long))))resolve-visible-column)fields)- query (as-> base-query query- (reduce (fn [query [expr-or-column expr-name]]- (lib/expression query expr-name expr-or-column))- query- (filter (comp expression? first) projection))- (add-fields query projection)- (reduce add-filter query (map resolve-visible-column filters))- (reduce add-aggregation query (map resolve-visible-column aggregations))- (reduce add-breakout query (map resolve-visible-column group-by))- (reduce add-order-by query (map resolve-order-by-column order-by))- (add-limit query limit))+ ;; Measures and segments don't require column resolution+ all-aggregations (map #(if (:measure-id %) % (resolve-visible-column %)) aggregations)+ all-filters (map #(if (:segment-id %) % (resolve-visible-column %)) filters)+ reduce-query (fn [query f coll] (reduce f query coll))+ query (-> base-query+ (reduce-query (fn [query [expr-or-column expr-name]]+ (lib/expression query expr-name expr-or-column))+ (filter (comp expression? first) projection))+ (add-fields projection)+ (reduce-query add-filter all-filters)+ (reduce-query add-aggregation all-aggregations)+ (reduce-query add-breakout (map resolve-visible-column group-by))+ (reduce-query add-order-by (map resolve-order-by-column order-by))+ (add-limit limit))query-id (u/generate-nano-id)query-field-id-prefix (metabot-v3.tools.u/query-field-id-prefix query-id)returned-cols (lib/returned-columns query)]{:type :query:query-id query-id- ;; existing usage, don't do this going forward -- use Lib instead and persist MBQL 5 to the app DB- :query #_{:clj-kondo/ignore [:discouraged-var]} (lib/->legacy-MBQL query)+ :query query:result-columns (into [](map-indexed #(metabot-v3.tools.u/->result-column query %2 %1 query-field-id-prefix))returned-cols)}))@@ -380,7 +409,7 @@(defn filter-records"Add `filters` to the query referenced by `data-source`"- [{:keys [data-source filters]}]+ [{:keys [data-source filters] :as _arguments}](try(let [[filter-field-id-prefix base] (base-query data-source)returned-cols (lib/returned-columns base)@@ -390,8 +419,7 @@{:structured-output{:type :query:query-id query-id- ;; existing usage, don't do this going forward -- use Lib instead and persist MBQL 5 to the app DB- :query #_{:clj-kondo/ignore [:discouraged-var]} (lib/->legacy-MBQL query)+ :query query:result-columns (into [](map-indexed #(metabot-v3.tools.u/->result-column query %2 %1 query-field-id-prefix))(lib/returned-columns query))}})diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.cljindex 60f1a576316..ffa116ac318 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj@@ -12,12 +12,13 @@[metabase-enterprise.metabot-v3.tools.filters :as metabot-v3.tools.filters][metabase-enterprise.metabot-v3.tools.find-outliers :as metabot-v3.tools.find-outliers][metabase-enterprise.metabot-v3.tools.generate-insights :as metabot-v3.tools.generate-insights]+ [metabase-enterprise.metabot-v3.tools.test-util :as tools.tu][metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u][metabase-enterprise.metabot-v3.util :as metabot-v3.u]- [metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.options :as lib.options][metabase.test :as mt][metabase.util :as u][metabase.util.malli.registry :as mr]@@ -249,21 +250,26 @@:filters filters:group_by breakouts}:conversation_id conversation-id})- query-id (-> response :structured_output :query_id)]+ query-id (-> response :structured_output :query_id)+ actual-query (-> response :structured_output :query lib-be/normalize-query)+ id-col (lib.metadata/field mp (mt/id :products :id))+ title-col (lib.metadata/field mp (mt/id :products :title))+ rating-col (lib.metadata/field mp (mt/id :products :rating))+ created-at-col (lib.metadata/field mp (mt/id :products :created_at))+ expected-query (-> (lib/query mp (lib.metadata/table mp (mt/id :products)))+ (lib/aggregate (lib.options/ensure-uuid [:metric {} metric-id]))+ (lib/breakout (lib/with-temporal-bucket created-at-col :week))+ (lib/breakout (lib/with-temporal-bucket created-at-col :day))+ (lib/filter (lib/> id-col 50))+ (lib/filter (lib/= title-col "3" "4"))+ (lib/filter (lib/!= rating-col 3 4))+ (lib/filter (lib/= (lib/get-month created-at-col) 4 5 9))+ (lib/filter (lib/!= (lib/get-day created-at-col) 14 15 19))+ (lib/filter (lib/= (lib/get-day-of-week created-at-col :iso) 1 7))+ (lib/filter (lib/= (lib/get-year created-at-col) 2008)))](is (=? {:structured_output{:type "query":query_id string?- :query (mt/mbql-query products- {:aggregation [[:metric metric-id]]- :breakout [!week.products.created_at !day.products.created_at]- :filter [:and- [:> [:field %id {}] 50]- [:= [:field %title {}] "3" "4"]- [:!= [:field %rating {}] 3 4]- [:= [:get-month [:field %created_at {}]] 4 5 9]- [:!= [:get-day [:field %products.created_at {}]] 14 15 19]- [:= [:get-day-of-week [:field %created_at {}] :iso] 1 7]- [:= [:get-year [:field %created_at {}]] 2008]]}):result_columns[{:field_id (str "q" query-id "-0"):name "CREATED_AT"@@ -284,8 +290,8 @@:database_type missing-value:semantic_type "score"}]}:conversation_id conversation-id}- (-> response- (update-in [:structured_output :query] mbql.normalize/normalize)))))))))+ response))+ (is (tools.tu/query= expected-query actual-query)))))))(deftest query-model-test(mt/with-premium-features #{:metabot-v3}diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/entity_details_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/entity_details_test.cljindex 47ca9be27df..072bdf48ea3 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/entity_details_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/entity_details_test.clj@@ -3,6 +3,7 @@[clojure.test :refer :all][metabase-enterprise.metabot-v3.tools.entity-details :as entity-details][metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.test :as mt][metabase.test.fixtures :as fixtures]))@@ -157,3 +158,119 @@(count (:fields products-related)))(str "Products related table should have " expected-products-field-count" fields (only its own fields, not implicitly joinable fields)"))))))))++(defn- measure-definition+ "Create an MBQL5 measure definition with a sum aggregation."+ [table-id field-id]+ (let [mp (mt/metadata-provider)+ table (lib.metadata/table mp table-id)+ query (lib/query mp table)+ field (lib.metadata/field mp field-id)]+ (lib/aggregate query (lib/sum field))))++(defn- segment-definition+ "Create an MBQL5 segment definition with a filter."+ [table-id field-id value]+ (let [mp (mt/metadata-provider)+ table (lib.metadata/table mp table-id)+ query (lib/query mp table)+ field (lib.metadata/field mp field-id)]+ (lib/filter query (lib/> field value))))++(deftest get-table-details-with-measures-test+ (testing "get-table-details returns measures when with_measures is true"+ (let [measure-def (measure-definition (mt/id :orders) (mt/id :orders :total))]+ (mt/with-temp [:model/Measure {measure-id :id} {:name "Total Revenue"+ :table_id (mt/id :orders)+ :definition measure-def}]+ (mt/with-current-user (mt/user->id :crowberto)+ (testing "with_measures: false (default) does not include measures"+ (let [result (entity-details/get-table-details {:table-id (mt/id :orders)})+ output (:structured-output result)]+ (is (nil? (:measures output)))))++ (testing "with_measures: true includes measures for the table"+ (let [result (entity-details/get-table-details {:table-id (mt/id :orders)+ :with-measures? true})+ output (:structured-output result)+ measures (:measures output)]+ (is (sequential? measures))+ (is (= 1 (count measures)))+ (let [measure (first measures)]+ (is (= measure-id (:id measure)))+ (is (= "Total Revenue" (:name measure)))+ (is (map? (:definition measure)))))))))))++(deftest get-table-details-with-segments-test+ (testing "get-table-details returns segments when with_segments is true"+ (let [segment-def (segment-definition (mt/id :orders) (mt/id :orders :total) 100)]+ (mt/with-temp [:model/Segment {segment-id :id} {:name "High Value Orders"+ :table_id (mt/id :orders)+ :definition segment-def}]+ (mt/with-current-user (mt/user->id :crowberto)+ (testing "with_segments: false (default) does not include segments"+ (let [result (entity-details/get-table-details {:table-id (mt/id :orders)})+ output (:structured-output result)]+ (is (nil? (:segments output)))))++ (testing "with_segments: true includes segments for the table"+ (let [result (entity-details/get-table-details {:table-id (mt/id :orders)+ :with-segments? true})+ output (:structured-output result)+ segments (:segments output)]+ (is (sequential? segments))+ (is (= 1 (count segments)))+ (let [segment (first segments)]+ (is (= segment-id (:id segment)))+ (is (= "High Value Orders" (:name segment)))+ (is (map? (:definition segment)))))))))))++(deftest get-table-details-measures-scoped-to-table-test+ (testing "get-table-details only returns measures for the requested table, not other tables"+ (let [orders-measure-def (measure-definition (mt/id :orders) (mt/id :orders :total))+ products-measure-def (measure-definition (mt/id :products) (mt/id :products :price))]+ (mt/with-temp [:model/Measure {orders-measure-id :id} {:name "Orders Total"+ :table_id (mt/id :orders)+ :definition orders-measure-def}+ :model/Measure {_products-measure-id :id} {:name "Products Price"+ :table_id (mt/id :products)+ :definition products-measure-def}]+ (mt/with-current-user (mt/user->id :crowberto)+ (let [result (entity-details/get-table-details {:table-id (mt/id :orders)+ :with-measures? true})+ output (:structured-output result)+ measures (:measures output)]+ (is (= 1 (count measures)))+ (is (= orders-measure-id (:id (first measures))))))))))++(deftest get-metric-details-with-segments-test+ (testing "get-metric-details returns segments when with_segments is true"+ ;; Note: lib/available-segments requires the query to have a direct source-table-id.+ ;; For metrics, this means the underlying card must be based on a table (not another card).+ (let [mp (mt/metadata-provider)+ metric-query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/aggregate (lib/sum (lib.metadata/field mp (mt/id :orders :total)))))+ segment-def (segment-definition (mt/id :orders) (mt/id :orders :total) 50)]+ (mt/with-temp [:model/Card {metric-id :id} {:dataset_query metric-query+ :database_id (mt/id)+ :name "Total Orders"+ :type :metric}+ :model/Segment {segment-id :id} {:name "Large Orders"+ :table_id (mt/id :orders)+ :definition segment-def}]+ (mt/with-current-user (mt/user->id :crowberto)+ (testing "with_segments: false (default) does not include segments"+ (let [result (entity-details/get-metric-details {:metric-id metric-id})+ output (:structured-output result)]+ (is (nil? (:segments output)))))++ (testing "with_segments: true includes segments for the metric"+ (let [result (entity-details/get-metric-details {:metric-id metric-id+ :with-segments? true})+ output (:structured-output result)+ segments (:segments output)]+ (is (sequential? segments))+ (is (= 1 (count segments)))+ (let [segment (first segments)]+ (is (= segment-id (:id segment)))+ (is (= "Large Orders" (:name segment)))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/filters_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/filters_test.cljindex 869ecd9daef..97970ed373c 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/filters_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/filters_test.clj@@ -1,12 +1,17 @@(ns metabase-enterprise.metabot-v3.tools.filters-test(:require+ [clojure.string :as str][clojure.test :refer :all][medley.core :as m][metabase-enterprise.metabot-v3.tools.entity-details :as metabot-v3.tools.entity-details][metabase-enterprise.metabot-v3.tools.filters :as metabot-v3.tools.filters]+ [metabase-enterprise.metabot-v3.tools.test-util :as tools.tu]+ [metabase.lib-be.core :as lib-be][metabase.lib.convert :as lib.convert][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.options :as lib.options]+ [metabase.lib.test-util :as lib.tu][metabase.test :as mt][metabase.util :as u]))@@ -42,35 +47,32 @@(when-not <>(throw (ex-info (str "Column " % " not found") {:column %}))))](testing "Trivial query works."- (is (=? {:structured-output {:type :query,- :query-id string?- :query {:database (mt/id)- :type :query- :query {:source-table (mt/id :orders)- :aggregation [[:metric metric-id]]}}}}- (metabot-v3.tools.filters/query-metric- {:metric-id metric-id- :filters []- :group-by []}))))+ (let [result (metabot-v3.tools.filters/query-metric+ {:metric-id metric-id+ :filters []+ :group-by []})+ actual-query (get-in result [:structured-output :query])+ expected-query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/aggregate (lib.options/ensure-uuid [:metric {} metric-id])))]+ (is (= :query (get-in result [:structured-output :type])))+ (is (string? (get-in result [:structured-output :query-id])))+ (is (tools.tu/query= expected-query actual-query))))(testing "Filtering and grouping works and ignores bucketing for non-temporal columns."(is (=? {:structured-output {:type :query,:query-id string?:query {:database (mt/id)- :type :query- :query {:source-table (mt/id :orders)- :aggregation [[:metric metric-id]]- :breakout [[:field (mt/id :products :category)- {:base-type :type/Text- :source-field (mt/id :orders :product_id)}]]- :filter- [:and- [:= [:field (mt/id :people :state)- {:base-type :type/Text- :source-field (mt/id :orders :user_id)}]- "TX"]- [:> [:field (mt/id :orders :discount)- {:base-type :type/Float}]- 3]]}}}}+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-table (mt/id :orders)+ :aggregation [[:metric {} metric-id]]+ :breakout [[:field {:source-field (mt/id :orders :product_id)}+ (mt/id :products :category)]]+ :filters+ [[:= {} [:field {:source-field (mt/id :orders :user_id)}+ (mt/id :people :state)]+ "TX"]+ [:> {} [:field {} (mt/id :orders :discount)]+ 3]]}]}}}(metabot-v3.tools.filters/query-metric{:metric-id metric-id:filters [{:field-id (->field-id ["User" "State"])@@ -85,23 +87,18 @@(is (=? {:structured-output {:type :query,:query-id string?:query {:database (mt/id)- :type :query- :query {:source-table (mt/id :orders)- :aggregation [[:metric metric-id]]- :filter [:and- [:!=- [:get-week- [:field (mt/id :orders :created_at)- {:base-type :type/DateTimeWithLocalTZ}]- :iso]- 1 2 3]- [:=- [:get-month [:field (mt/id :orders :created_at)- {:base-type :type/DateTimeWithLocalTZ}]]- 6 7 8]]- :breakout [[:field (mt/id :orders :created_at)- {:base-type :type/DateTimeWithLocalTZ- :temporal-unit :week}]]}}}}+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-table (mt/id :orders)+ :aggregation [[:metric {} metric-id]]+ :filters [[:!= {}+ [:get-week {} [:field {} (mt/id :orders :created_at)] :iso]+ 1 2 3]+ [:= {}+ [:get-month {} [:field {} (mt/id :orders :created_at)]]+ 6 7 8]]+ :breakout [[:field {:temporal-unit :week}+ (mt/id :orders :created_at)]]}]}}}(metabot-v3.tools.filters/query-metric{:metric-id metric-id:filters [{:field-id (->field-id "Created At")@@ -115,29 +112,34 @@:group-by [{:field-id (->field-id "Created At"):field-granularity :week}]}))))(testing "Multi-value filtering works"- (let [state-id (->field-id ["User" "State"])- stat-field-clause [:field (mt/id :people :state)- {:base-type :type/Text- :source-field (mt/id :orders :user_id)}]]+ (let [state-id (->field-id ["User" "State"])](is (=? {:structured-output {:type :query,:query-id string?:query {:database (mt/id)- :type :query- :query {:source-table (mt/id :orders)- :aggregation [[:metric metric-id]]- :filter- [:and- [:contains {:case-sensitive false}- stat-field-clause "o" "e"]- [:does-not-contain- stat-field-clause "y" {:case-sensitive false}]- [:starts-with {:case-sensitive false}- stat-field-clause "A" "G"]- [:ends-with {:case-sensitive false}- stat-field-clause "e" "f" "i" "T" "x"]- [:!=- [:field (mt/id :orders :discount) {:base-type :type/Float}]- 3 42]]}}}}+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-table (mt/id :orders)+ :aggregation [[:metric {} metric-id]]+ :filters+ [[:contains {:case-sensitive false}+ [:field {:source-field (mt/id :orders :user_id)}+ (mt/id :people :state)]+ "o" "e"]+ [:does-not-contain {:case-sensitive false}+ [:field {:source-field (mt/id :orders :user_id)}+ (mt/id :people :state)]+ "y"]+ [:starts-with {:case-sensitive false}+ [:field {:source-field (mt/id :orders :user_id)}+ (mt/id :people :state)]+ "A" "G"]+ [:ends-with {:case-sensitive false}+ [:field {:source-field (mt/id :orders :user_id)}+ (mt/id :people :state)]+ "e" "f" "i" "T" "x"]+ [:!= {}+ [:field {} (mt/id :orders :discount)]+ 3 42]]}]}}}(metabot-v3.tools.filters/query-metric{:metric-id metric-id:filters [{:field-id state-id@@ -177,7 +179,6 @@(mt/with-current-user (mt/user->id :crowberto)(let [model-details (-> (metabot-v3.tools.entity-details/get-table-details {:model-id model-id}):structured-output)- model-card-id (str "card__" model-id)->field-id #(u/prog1 (-> model-details :fields (by-name %) :field_id)(when-not <>(throw (ex-info (str "Column " % " not found") {:column %}))))@@ -185,7 +186,10 @@(testing "Trivial query works."(is (=? {:structured-output {:type :query,:query-id string?- :query (mt/mbql-query orders {:source-table model-card-id})}}+ :query {:database (mt/id)+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-card model-id}]}}}(metabot-v3.tools.filters/query-model{:model-id model-id:filters []@@ -193,11 +197,13 @@(testing "Filtering, aggregation and grouping works and ignores bucketing for non-temporal columns."(is (=? {:structured-output {:type :query,:query-id string?- :query (mt/mbql-query orders- {:source-table model-card-id- :aggregation [[:sum [:field "SUBTOTAL" {}]]]- :breakout [[:field "PRODUCT_ID" {}]]- :filter [:> [:field "DISCOUNT" {}] 3]})}}+ :query {:database (mt/id)+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-card model-id+ :aggregation [[:sum {} [:field {} "SUBTOTAL"]]]+ :breakout [[:field {} "PRODUCT_ID"]]+ :filters [[:> {} [:field {} "DISCOUNT"] 3]]}]}}}(metabot-v3.tools.filters/query-model{:model-id model-id:filters [{:field-id (->field-id "Discount")@@ -211,27 +217,17 @@(testing "Temporal bucketing works for temporal columns."(is (=? {:structured-output {:type :query,:query-id string?- :query (mt/mbql-query orders- {:source-table model-card-id- :aggregation [[:min [:field "CREATED_AT"- {:base-type :type/DateTimeWithLocalTZ- :temporal-unit :hour-of-day}]]- [:avg [:field "CREATED_AT"- {:base-type :type/DateTimeWithLocalTZ- :temporal-unit :hour-of-day}]]- [:max [:field "CREATED_AT"- {:base-type :type/DateTimeWithLocalTZ- :temporal-unit :hour-of-day}]]]- :filter [:and- [:!= [:get-week [:field "CREATED_AT" {}] :iso] 1 2 3]- [:=- [:get-month [:field "CREATED_AT"- {:base-type :type/DateTimeWithLocalTZ}]]- 6 7 8]]- :breakout [[:field "PRODUCT_ID" {}]- [:field "CREATED_AT"- {:base-type :type/DateTimeWithLocalTZ- :temporal-unit :week}]]})}}+ :query {:database (mt/id)+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-card model-id+ :aggregation [[:min {} [:field {:temporal-unit :hour-of-day} "CREATED_AT"]]+ [:avg {} [:field {:temporal-unit :hour-of-day} "CREATED_AT"]]+ [:max {} [:field {:temporal-unit :hour-of-day} "CREATED_AT"]]]+ :filters [[:!= {} [:get-week {} [:field {} "CREATED_AT"] :iso] 1 2 3]+ [:= {} [:get-month {} [:field {} "CREATED_AT"]] 6 7 8]]+ :breakout [[:field {} "PRODUCT_ID"]+ [:field {:temporal-unit :week} "CREATED_AT"]]}]}}}(metabot-v3.tools.filters/query-model{:model-id model-id:filters [{:field-id order-created-at-field-id@@ -287,9 +283,11 @@(let [expected-query {:structured-output{:type :query,:query-id string?- :query (mt/mbql-query orders- {:source-table model-card-id- :filter [:!= [:field "USER_ID" {}] 3 42]})}}+ :query {:database (mt/id)+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-card model-id+ :filters [[:!= {} [:field {} "USER_ID"] 3 42]]}]}}}input {:model-id model-id:filters [{:field-id (->field-id "User ID"):operation :not-equals@@ -319,40 +317,34 @@(when-not <>(throw (ex-info (str "Column " % " not found") {:column %}))))](testing "Trivial query works."- (is (=? {:structured-output {:type :query,- :query-id string?- :query {:database (mt/id)- :type :query- :query {:source-table table-id}}}}- (metabot-v3.tools.filters/filter-records- {:data-source {:table-id table-id}- :filters []}))))+ (let [result (metabot-v3.tools.filters/filter-records+ {:data-source {:table-id table-id}+ :filters []})+ actual-query (get-in result [:structured-output :query])+ expected-query (lib/query mp (lib.metadata/table mp table-id))]+ (is (= :query (get-in result [:structured-output :type])))+ (is (string? (get-in result [:structured-output :query-id])))+ (is (tools.tu/query= expected-query actual-query))))(testing "Filtering works."- (is (=? {:structured-output {:type :query,- :query-id string?- :query {:database (mt/id)- :type :query- :query {:source-table (mt/id :orders)- :filter- [:and- [:=- [:get-day-of-week- [:field (mt/id :orders :created_at)- {:base-type :type/DateTimeWithLocalTZ}]- :iso]- 1 7]- [:>- [:field (mt/id :orders :discount) {:base-type :type/Float}]- 3]]}}}}- (metabot-v3.tools.filters/filter-records- {:data-source {:table-id table-id}- :filters [{:field-id (->field-id "Created At")- :bucket :day-of-week- :operation :equals- :values [1 7]}- {:field-id (->field-id "Discount")- :operation :number-greater-than- :value 3}]})))))+ (let [result (metabot-v3.tools.filters/filter-records+ {:data-source {:table-id table-id}+ :filters [{:field-id (->field-id "Created At")+ :bucket :day-of-week+ :operation :equals+ :values [1 7]}+ {:field-id (->field-id "Discount")+ :operation :number-greater-than+ :value 3}]})+ actual-query (get-in result [:structured-output :query])+ orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ created-at-col (lib.metadata/field mp (mt/id :orders :created_at))+ discount-col (lib.metadata/field mp (mt/id :orders :discount))+ expected-query (-> orders-query+ (lib/filter (lib/= (lib/get-day-of-week created-at-col :iso) 1 7))+ (lib/filter (lib/> discount-col 3)))]+ (is (= :query (get-in result [:structured-output :type])))+ (is (string? (get-in result [:structured-output :query-id])))+ (is (tools.tu/query= expected-query actual-query)))))(testing "Missing table results in an error."(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Not found."(metabot-v3.tools.filters/filter-records@@ -382,8 +374,9 @@(is (=? {:structured-output {:type :query,:query-id string?:query {:database (mt/id)- :type :query- :query {:source-table table-id}}}}+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-card model-id}]}}}(metabot-v3.tools.filters/filter-records{:data-source {:table-id table-id}:filters []}))))@@ -391,9 +384,10 @@(is (=? {:structured-output {:type :query,:query-id string?:query {:database (mt/id)- :type :query- :query {:source-table table-id- :filter [:> [:field "DISCOUNT" {:base-type :type/Float}] 3]}}}}+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-card model-id+ :filters [[:> {} [:field {} "DISCOUNT"] 3]]}]}}}(metabot-v3.tools.filters/filter-records{:data-source {:table-id table-id}:filters [{:field-id (->field-id "Discount")@@ -428,8 +422,9 @@(is (=? {:structured-output {:type :query,:query-id string?:query {:database (mt/id)- :type :query- :query {:source-table table-id}}}}+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-table table-id}]}}}(metabot-v3.tools.filters/filter-records{:data-source {:report-id card-id}:filters []}))))@@ -437,13 +432,10 @@(is (=? {:structured-output {:type :query,:query-id string?:query {:database (mt/id)- :type :query- :query {:source-table table-id- :filter [:>- [:field- (mt/id :orders :discount)- {:base-type :type/Float}]- 3]}}}}+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-table table-id+ :filters [[:> {} [:field {} (mt/id :orders :discount)] 3]]}]}}}(metabot-v3.tools.filters/filter-records{:data-source {:report-id card-id}:filters [{:field-id (->field-id "Discount")@@ -472,13 +464,11 @@expected {:structured-output {:type :query,:query-id string?:query {:database (mt/id)- :type :query- :query {:source-query {:source-table table-id}- :filter [:>- [:field- "DISCOUNT"- {:base-type :type/Float}]- 3]}}}}]+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-table table-id}+ {:lib/type :mbql.stage/mbql+ :filters [[:> {} [:field {} "DISCOUNT"] 3]]}]}}}](mt/with-current-user (mt/user->id :crowberto)(testing "Filtering works."(testing "new tool call with query and query-id"@@ -506,8 +496,9 @@(is (=? {:structured-output {:type :query:query-id string?:query {:database (mt/id)- :type :query- :query {:source-table table-id}}}}+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-table table-id}]}}}(metabot-v3.tools.filters/query-datasource{:table-id table-id}))))@@ -515,10 +506,11 @@(is (=? {:structured-output {:type :query:query-id string?:query {:database (mt/id)- :type :query- :query {:source-table table-id- :fields [[:field (mt/id :orders :created_at) {:base-type :type/DateTimeWithLocalTZ}]- [:field (mt/id :orders :total) {:base-type :type/Float}]]}}}}+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-table table-id+ :fields [[:field {} (mt/id :orders :created_at)]+ [:field {} (mt/id :orders :total)]]}]}}}(metabot-v3.tools.filters/query-datasource{:table-id table-id:fields [{:field-id (->field-id "Created At")}@@ -528,11 +520,11 @@(is (=? {:structured-output {:type :query:query-id string?:query {:database (mt/id)- :type :query- :query {:source-table table-id- :filter [:and- [:> [:field (mt/id :orders :discount) {:base-type :type/Float}] 3]- [:= [:field (mt/id :orders :user_id) {:base-type :type/Integer}] 10]]}}}}+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-table table-id+ :filters [[:> {} [:field {} (mt/id :orders :discount)] 3]+ [:= {} [:field {} (mt/id :orders :user_id)] 10]]}]}}}(metabot-v3.tools.filters/query-datasource{:table-id table-id:filters [{:field-id (->field-id "Discount")@@ -546,11 +538,12 @@(is (=? {:structured-output {:type :query:query-id string?:query {:database (mt/id)- :type :query- :query {:source-table table-id- :aggregation [[:sum [:field (mt/id :orders :subtotal) {:base-type :type/Float}]]- [:avg [:field (mt/id :orders :discount) {:base-type :type/Float}]]]- :breakout [[:field (mt/id :orders :product_id) {:base-type :type/Integer}]]}}}}+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-table table-id+ :aggregation [[:sum {} [:field {} (mt/id :orders :subtotal)]]+ [:avg {} [:field {} (mt/id :orders :discount)]]]+ :breakout [[:field {} (mt/id :orders :product_id)]]}]}}}(metabot-v3.tools.filters/query-datasource{:table-id table-id:aggregations [{:field-id (->field-id "Subtotal")@@ -563,11 +556,12 @@(is (=? {:structured-output {:type :query:query-id string?:query {:database (mt/id)- :type :query- :query {:source-table table-id- :order-by [[:asc [:field (mt/id :orders :created_at) {:base-type :type/DateTimeWithLocalTZ}]]- [:desc [:field (mt/id :orders :total) {:base-type :type/Float}]]]- :limit 100}}}}+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-table table-id+ :order-by [[:asc {} [:field {} (mt/id :orders :created_at)]]+ [:desc {} [:field {} (mt/id :orders :total)]]]+ :limit 100}]}}}(metabot-v3.tools.filters/query-datasource{:table-id table-id:order-by [{:field {:field-id (->field-id "Created At")}@@ -580,12 +574,11 @@(is (=? {:structured-output {:type :query:query-id string?:query {:database (mt/id)- :type :query- :query {:source-table table-id- :aggregation [[:count]]- :breakout [[:field (mt/id :orders :created_at)- {:base-type :type/DateTimeWithLocalTZ- :temporal-unit :month}]]}}}}+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-table table-id+ :aggregation [[:count {}]]+ :breakout [[:field {:temporal-unit :month} (mt/id :orders :created_at)]]}]}}}(metabot-v3.tools.filters/query-datasource{:table-id table-id:aggregations [{:field-id (->field-id "Product ID")@@ -602,24 +595,28 @@(mt/with-current-user (mt/user->id :crowberto)(let [model-details (-> (metabot-v3.tools.entity-details/get-table-details {:model-id model-id}):structured-output)- model-card-id (str "card__" model-id)->field-id #(u/prog1 (-> model-details :fields (by-name %) :field_id)(when-not <>(throw (ex-info (str "Column " % " not found") {:column %}))))](testing "Basic model query works"(is (=? {:structured-output {:type :query:query-id string?- :query (mt/mbql-query orders {:source-table model-card-id})}}+ :query {:database (mt/id)+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-card model-id}]}}}(metabot-v3.tools.filters/query-datasource{:model-id model-id}))))(testing "Model query with fields selection"(is (=? {:structured-output {:type :query:query-id string?- :query (mt/mbql-query orders- {:source-table model-card-id- :fields [[:field "CREATED_AT" {:base-type :type/DateTimeWithLocalTZ}]- [:field "TOTAL" {:base-type :type/Float}]]})}}+ :query {:database (mt/id)+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-card model-id+ :fields [[:field {} "CREATED_AT"]+ [:field {} "TOTAL"]]}]}}}(metabot-v3.tools.filters/query-datasource{:model-id model-id:fields [{:field-id (->field-id "Created At")}@@ -628,11 +625,12 @@(testing "Model query with filters"(is (=? {:structured-output {:type :query:query-id string?- :query (mt/mbql-query orders- {:source-table model-card-id- :filter [:and- [:> [:field "DISCOUNT" {:base-type :type/Float}] 5]- [:< [:field "SUBTOTAL" {:base-type :type/Float}] 100]]})}}+ :query {:database (mt/id)+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-card model-id+ :filters [[:> {} [:field {} "DISCOUNT"] 5]+ [:< {} [:field {} "SUBTOTAL"] 100]]}]}}}(metabot-v3.tools.filters/query-datasource{:model-id model-id:filters [{:field-id (->field-id "Discount")@@ -645,11 +643,13 @@(testing "Model query with aggregations and grouping"(is (=? {:structured-output {:type :query:query-id string?- :query (mt/mbql-query orders- {:source-table model-card-id- :aggregation [[:count]- [:max [:field "TOTAL" {:base-type :type/Float}]]]- :breakout [[:field "USER_ID" {:base-type :type/Integer}]]})}}+ :query {:database (mt/id)+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-card model-id+ :aggregation [[:count {}]+ [:max {} [:field {} "TOTAL"]]]+ :breakout [[:field {} "USER_ID"]]}]}}}(metabot-v3.tools.filters/query-datasource{:model-id model-id:aggregations [{:field-id (->field-id "Product ID")@@ -661,10 +661,12 @@(testing "Model query with order by and limit"(is (=? {:structured-output {:type :query:query-id string?- :query (mt/mbql-query orders- {:source-table model-card-id- :order-by [[:desc [:field "CREATED_AT" {:base-type :type/DateTimeWithLocalTZ}]]]- :limit 50})}}+ :query {:database (mt/id)+ :lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :source-card model-id+ :order-by [[:desc {} [:field {} "CREATED_AT"]]]+ :limit 50}]}}}(metabot-v3.tools.filters/query-datasource{:model-id model-id:order-by [{:field {:field-id (->field-id "Created At")}@@ -717,3 +719,170 @@(is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."(metabot-v3.tools.filters/query-datasource{:model-id model-id}))))))++(defn- mbql-measure-definition+ "Create an MBQL5 measure definition with a sum aggregation."+ [table-id field-id]+ (let [mp (mt/metadata-provider)+ table (lib.metadata/table mp table-id)+ query (lib/query mp table)+ field (lib.metadata/field mp field-id)]+ (lib/aggregate query (lib/sum field))))++(defn- mbql-segment-definition+ "Create an MBQL5 segment definition with a filter."+ [table-id field-id value]+ (let [mp (mt/metadata-provider)+ table (lib.metadata/table mp table-id)+ query (lib/query mp table)+ field (lib.metadata/field mp field-id)]+ (lib/filter query (lib/> field value))))++(defn- add-mock-segment+ "Add a mock segment to a metadata provider."+ [base-mp segment-id table-id definition]+ (lib.tu/mock-metadata-provider+ base-mp+ {:segments [{:lib/type :metadata/segment+ :id segment-id+ :name (str "Mock Segment " segment-id)+ :table-id table-id+ :archived false+ :definition definition}]}))++(defn- add-mock-measure+ "Add a mock measure to a metadata provider."+ [base-mp measure-id table-id definition]+ (lib.tu/mock-metadata-provider+ base-mp+ {:measures [{:lib/type :metadata/measure+ :id measure-id+ :name (str "Mock Measure " measure-id)+ :table-id table-id+ :archived false+ :definition definition}]}))++(deftest query-metric-with-segment-filter-test+ (let [mp (mt/metadata-provider)+ created-at-meta (lib.metadata/field mp (mt/id :orders :created_at))+ metric-query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/aggregate (lib/sum (lib.metadata/field mp (mt/id :orders :total))))+ (lib/breakout (lib/with-temporal-bucket created-at-meta :month)))+ legacy-metric-query (lib.convert/->legacy-MBQL metric-query)+ segment-def (mbql-segment-definition (mt/id :orders) (mt/id :orders :total) 50)+ mock-segment-id 1+ mock-mp (add-mock-segment mp mock-segment-id (mt/id :orders) segment-def)]+ (mt/with-temp [:model/Card {metric-id :id} {:dataset_query legacy-metric-query+ :database_id (mt/id)+ :name "Total Orders"+ :type :metric}]+ (mt/with-current-user (mt/user->id :crowberto)+ (testing "query-metric with segment filter applies the segment"+ (with-redefs [lib-be/application-database-metadata-provider (constantly mock-mp)]+ (let [result (metabot-v3.tools.filters/query-metric+ {:metric-id metric-id+ :filters [{:segment-id mock-segment-id}]+ :group-by []})+ query (get-in result [:structured-output :query])+ filters (get-in query [:stages 0 :filters])]+ (is (some? (:structured-output result)))+ (is (= :query (get-in result [:structured-output :type])))+ (is (=? [[:segment {} mock-segment-id]] filters)))))))))++(deftest query-datasource-with-measure-aggregation-test+ (let [mp (mt/metadata-provider)+ measure-def (mbql-measure-definition (mt/id :orders) (mt/id :orders :total))+ mock-measure-id 1+ mock-mp (add-mock-measure mp mock-measure-id (mt/id :orders) measure-def)]+ (mt/with-current-user (mt/user->id :crowberto)+ (with-redefs [lib-be/application-database-metadata-provider (constantly mock-mp)]+ (testing "query-datasource with measure aggregation"+ (let [result (metabot-v3.tools.filters/query-datasource+ {:table-id (mt/id :orders)+ :aggregations [{:measure-id mock-measure-id}]})+ query (get-in result [:structured-output :query])+ aggregations (get-in query [:stages 0 :aggregation])]+ (is (some? (:structured-output result)))+ (is (= :query (get-in result [:structured-output :type])))+ ;; Check that a measure aggregation is present+ (is (some #(and (vector? %) (= :measure (first %))) aggregations))))++ (testing "query-datasource with measure aggregation and sort order"+ (let [result (metabot-v3.tools.filters/query-datasource+ {:table-id (mt/id :orders)+ :aggregations [{:measure-id mock-measure-id+ :sort-order :desc}]})+ query (get-in result [:structured-output :query])+ order-by (get-in query [:stages 0 :order-by])]+ (is (some? order-by))+ (is (some #(= :desc (first %)) order-by))))))))++(deftest query-datasource-with-segment-filter-test+ (let [mp (mt/metadata-provider)+ segment-def (mbql-segment-definition (mt/id :orders) (mt/id :orders :total) 100)+ mock-segment-id 1+ mock-mp (add-mock-segment mp mock-segment-id (mt/id :orders) segment-def)]+ (mt/with-current-user (mt/user->id :crowberto)+ (with-redefs [lib-be/application-database-metadata-provider (constantly mock-mp)]+ (testing "query-datasource with segment filter"+ (let [result (metabot-v3.tools.filters/query-datasource+ {:table-id (mt/id :orders)+ :filters [{:segment-id mock-segment-id}]})+ query (get-in result [:structured-output :query])+ filters (get-in query [:stages 0 :filters])]+ (is (some? (:structured-output result)))+ (is (= :query (get-in result [:structured-output :type])))+ ;; Check that a segment filter is present+ (is (some #(and (vector? %) (= :segment (first %))) filters))))))))++(deftest query-datasource-with-measures-and-segments-test+ (let [mp (mt/metadata-provider)+ measure-def (mbql-measure-definition (mt/id :orders) (mt/id :orders :total))+ segment-def (mbql-segment-definition (mt/id :orders) (mt/id :orders :discount) 0)+ mock-measure-id 1+ mock-segment-id 1+ mock-mp (-> mp+ (add-mock-measure mock-measure-id (mt/id :orders) measure-def)+ (add-mock-segment mock-segment-id (mt/id :orders) segment-def))]+ (mt/with-current-user (mt/user->id :crowberto)+ (with-redefs [lib-be/application-database-metadata-provider (constantly mock-mp)]+ (testing "query-datasource with both measure aggregation and segment filter"+ (let [result (metabot-v3.tools.filters/query-datasource+ {:table-id (mt/id :orders)+ :aggregations [{:measure-id mock-measure-id}]+ :filters [{:segment-id mock-segment-id}]})+ query (get-in result [:structured-output :query])+ aggregations (get-in query [:stages 0 :aggregation])+ filters (get-in query [:stages 0 :filters])]+ (is (some? (:structured-output result)))+ ;; Check that both measure and segment are present+ (is (some #(and (vector? %) (= :measure (first %))) aggregations))+ (is (some #(and (vector? %) (= :segment (first %))) filters))))))))++(deftest segment-not-found-test+ (let [mp (mt/metadata-provider)+ non-existent-segment-id 99999]+ (mt/with-current-user (mt/user->id :crowberto)+ (with-redefs [lib-be/application-database-metadata-provider (constantly mp)]+ (testing "query-datasource with non-existent segment returns error"+ (let [result (metabot-v3.tools.filters/query-datasource+ {:table-id (mt/id :orders)+ :filters [{:segment-id non-existent-segment-id}]})]+ (is (nil? (:structured-output result)))+ (is (string? (:output result)))+ (is (str/includes? (:output result) "Segment"))+ (is (str/includes? (:output result) "not found"))))))))++(deftest measure-not-found-test+ (let [mp (mt/metadata-provider)+ non-existent-measure-id 99999]+ (mt/with-current-user (mt/user->id :crowberto)+ (with-redefs [lib-be/application-database-metadata-provider (constantly mp)]+ (testing "query-datasource with non-existent measure returns error"+ (let [result (metabot-v3.tools.filters/query-datasource+ {:table-id (mt/id :orders)+ :aggregations [{:measure-id non-existent-measure-id}]})]+ (is (nil? (:structured-output result)))+ (is (string? (:output result)))+ (is (str/includes? (:output result) "Measure"))+ (is (str/includes? (:output result) "not found"))))))))diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/test_util.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/test_util.cljnew file mode 100644index 00000000000..43fa71ec3ad--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/test_util.clj@@ -0,0 +1,9 @@+(ns metabase-enterprise.metabot-v3.tools.test-util+ (:require+ [metabase.lib.core :as lib]))++(defn query=+ "Compare two queries for equality, ignoring lib/uuids."+ [expected actual]+ (= (lib/remove-lib-uuids expected)+ (lib/remove-lib-uuids actual)))