Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Split fields returned to metabot for cards/models into core fields and related_tables (#64385)

Noah Moss4215b09ac7fb
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.clj
index 872887d4063..1ef36343f10 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.clj
@@ -126,17 +126,20 @@
(metric-details metadata-provider (assoc options :with-queryable-dimensions? false))
(select-keys [:id :type :name :description :default_time_dimension_field_id]))))
+(declare related-tables)
+
(defn- table-details
([id] (table-details id nil))
- ([id {:keys [metadata-provider field-values-fn with-fields? with-metrics?]
- :or {field-values-fn add-field-values
- with-fields? true
- with-metrics? true}
+ ([id {:keys [metadata-provider field-values-fn with-fields? with-related-tables? with-metrics?]
+ :or {field-values-fn add-field-values
+ with-fields? true
+ with-related-tables? true
+ with-metrics? true}
:as options}]
(when-let [base (if metadata-provider
(lib.metadata/table metadata-provider id)
(metabot-v3.tools.u/get-table id :db_id :description :name :schema))]
- (let [query-needed? (or with-fields? with-metrics?)
+ (let [query-needed? (or with-fields? with-related-tables? with-metrics?)
db-id (if metadata-provider (:db-id base) (:db_id base))
mp (when query-needed?
(or metadata-provider
@@ -148,7 +151,9 @@
field-values-fn
(map #(metabot-v3.tools.u/add-table-reference table-query %))))
field-id-prefix (when with-fields?
- (metabot-v3.tools.u/table-field-id-prefix id))]
+ (metabot-v3.tools.u/table-field-id-prefix id))
+ related-tables (when with-related-tables?
+ (related-tables table-query with-fields? field-values-fn))]
(-> {:id id
:type :table
:fields (into [] (map-indexed #(metabot-v3.tools.u/->result-column table-query %2 %1 field-id-prefix)) cols)
@@ -159,27 +164,43 @@
:database_id db-id
:database_schema (:schema base)}
(m/assoc-some :description (:description base)
+ :related_tables related-tables
:metrics (when with-metrics?
(not-empty (mapv #(convert-metric % mp options)
(lib/available-metrics table-query))))))))))
+(defn- related-tables
+ "Constructs a list of tables, optionally including their fields, that are related to the given query via foreign key."
+ [query with-fields? field-values-fn]
+ (let [fk-fields (->> (lib/visible-columns query)
+ (filter :fk-field-id))
+ related-table-ids (->> fk-fields
+ (map :table-id)
+ (into #{}))]
+ (when (seq related-table-ids)
+ (map #(table-details % {:with-fields? with-fields?
+ :field-values-fn field-values-fn
+ ;; Important! Only recurse one level
+ :with-related-tables? false
+ :with-metrics? false})
+ related-table-ids))))
+
(defn- card-details
"Get details for a card."
([id] (card-details id nil))
([id options]
(when-let [card (metabot-v3.tools.u/get-card id)]
(card-details card (lib.metadata.jvm/application-database-metadata-provider (:database_id card)) options)))
- ([base metadata-provider {:keys [field-values-fn with-fields? with-metrics?]
- :or {field-values-fn add-field-values
- with-fields? true
- with-metrics? true}
+ ([base metadata-provider {:keys [field-values-fn with-fields? with-related-tables? with-metrics?]
+ :or {field-values-fn add-field-values
+ with-fields? true
+ with-related-tables? true
+ with-metrics? true}
:as options}]
(let [id (:id base)
- query-needed? (or with-fields? with-metrics?)
- card-metadata (when query-needed?
- (lib.metadata/card metadata-provider id))
- dataset-query (when query-needed?
- (get card-metadata :dataset-query))
+ card-metadata (lib.metadata/card metadata-provider id)
+ dataset-query (get card-metadata :dataset-query)
+ query-needed? (or with-fields? with-related-tables? with-metrics?)
;; pivot questions have strange result-columns so we work with the dataset-query
card-type (:type base)
card-query (when query-needed?
@@ -188,25 +209,26 @@
(#{:query} (:type dataset-query)))
dataset-query
card-metadata)))
- cols (when with-fields?
- (->> (lib/visible-columns card-query)
- field-values-fn
- (map #(metabot-v3.tools.u/add-table-reference card-query %))))
+ returned-fields (when with-fields?
+ (->> (lib/returned-columns card-query)
+ field-values-fn))
+ related-tables (when with-related-tables?
+ (related-tables card-query with-fields? field-values-fn))
field-id-prefix (metabot-v3.tools.u/card-field-id-prefix id)]
(-> {:id id
:type card-type
- :fields (into [] (map-indexed #(metabot-v3.tools.u/->result-column card-query %2 %1 field-id-prefix)) cols)
+ :fields (into [] (map-indexed #(metabot-v3.tools.u/->result-column card-query %2 %1 field-id-prefix)) returned-fields)
:name (:name base)
:display_name (some->> (:name base)
(u.humanization/name->human-readable-name :simple))
:database_id (:database_id base)
- :queryable-foreign-key-tables []
:verified (verified-review? id "card")}
-
- (m/assoc-some :description (:description base)
- :metrics (when with-metrics?
- (not-empty (mapv #(convert-metric % metadata-provider options)
- (lib/available-metrics card-query)))))))))
+ (m/assoc-some
+ :description (:description base)
+ :related_tables related-tables
+ :metrics (when with-metrics?
+ (not-empty (mapv #(convert-metric % metadata-provider options)
+ (lib/available-metrics card-query)))))))))
(defn cards-details
"Get the details of metrics or models as specified by `card-type` and `cards`
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj
index eb20f4fd73d..1e045d86cc9 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj
@@ -583,6 +583,7 @@
[:table_id {:optional true} [:or :int :string]]
[:with_fields {:optional true, :default true} :boolean]
[:with_field_values {:optional true, :default true} :boolean]
+ [:with_related_tables {:optional true, :default true} :boolean]
[:with_metrics {:optional true, :default true} :boolean]
[:with_metric_default_temporal_breakout {:optional true, :default true} :boolean]]
[:fn {:error/message "Exactly one of model_id and table_id required"}
@@ -592,31 +593,30 @@
:table_id :table-id
:with_fields :with-fields?
:with_field_values :with-field-values?
+ :with_related_tables :with-related-tables?
:with_metrics :with-metrics?
:with_metric_default_temporal_breakout :with-default-temporal-breakout?})}]])
-(mr/def ::basic-table
- [:map
- [:id :int]
- [:type [:enum :model :table]]
- [:name :string]
- [:display_name :string]
- [:database_id :int]
- [:database_schema {:optional true} [:maybe :string]] ; Schema name, if applicable
- [:fields ::columns]
- [:description {:optional true} [:maybe :string]]
- [:metrics {:optional true} [:sequential ::basic-metric]]])
-
-(mr/def ::full-table
- [:merge
- ::basic-table
- [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
- [:queryable_foreign_key_tables {:optional true} [:sequential ::basic-table]]]])
+(mr/def ::table-result
+ [:schema
+ {:registry {::table-result
+ [:map
+ [:id :int]
+ [:type [:enum :model :table]]
+ [:name :string]
+ [:display_name :string]
+ [:database_id :int]
+ [:database_schema {:optional true} [:maybe :string]] ; Schema name, if applicable
+ [:fields ::columns]
+ [:related_tables {:optional true} [:sequential [:ref ::table-result]]]
+ [:description {:optional true} [:maybe :string]]
+ [:metrics {:optional true} [:sequential ::basic-metric]]]}}
+ ::table-result])
(mr/def ::get-table-details-result
[:or
[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
- [:structured_output ::full-table]]
+ [:structured_output ::table-result]]
[:map [:output :string]]])
(mr/def ::answer-sources-result
@@ -625,7 +625,7 @@
{:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}
[:structured_output [:map
[:metrics [:sequential ::full-metric]]
- [:models [:sequential ::full-table]]]]]
+ [:models [:sequential ::table-result]]]]]
[:map [:output :string]]])
(api.macros/defendpoint :post "/answer-sources" :- [:merge ::answer-sources-result ::tool-request]
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
index 352a83578f0..587f1c271c3 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
@@ -727,7 +727,8 @@
metric-data {:name "Metric"
:description "Model based metric"
:type :metric}
- expected-fields
+ ;; Core fields returned by the model (lib/returned-columns)
+ expected-core-fields
[{:name "ID", :display_name "ID", :type "number", :semantic_type "pk"}
{:name "USER_ID", :display_name "User ID", :type "number", :semantic_type "fk"}
{:name "PRODUCT_ID", :display_name "Product ID", :type "number", :semantic_type "fk"}
@@ -736,47 +737,59 @@
{:name "TOTAL", :display_name "Total", :type "number"}
{:name "DISCOUNT", :display_name "Discount", :type "number", :semantic_type "discount"}
{:name "CREATED_AT", :display_name "Created At", :type "datetime", :semantic_type "creation_timestamp"}
- {:name "QUANTITY", :display_name "Quantity", :type "number", :semantic_type "quantity", :field_values int-sequence?}
- {:name "ID", :display_name "ID", :type "number", :semantic_type "pk", :table_reference "User"}
- {:name "ADDRESS", :display_name "Address", :type "string", :table_reference "User"}
- {:name "EMAIL", :display_name "Email", :type "string", :semantic_type "email", :table_reference "User"}
- {:name "PASSWORD", :display_name "Password", :type "string", :table_reference "User"}
- {:name "NAME", :display_name "Name", :type "string", :semantic_type "name", :table_reference "User"}
- {:name "CITY", :display_name "City", :type "string", :semantic_type "city", :table_reference "User"}
- {:name "LONGITUDE", :display_name "Longitude", :type "number", :semantic_type "longitude", :table_reference "User"}
- {:name "STATE", :display_name "State", :type "string", :semantic_type "state", :table_reference "User"}
- {:name "SOURCE", :display_name "Source", :type "string", :semantic_type "source", :table_reference "User"}
- {:name "BIRTH_DATE", :display_name "Birth Date", :type "date", :table_reference "User"}
- {:name "ZIP", :display_name "Zip", :type "string", :table_reference "User"}
- {:name "LATITUDE", :display_name "Latitude", :type "number", :semantic_type "latitude", :table_reference "User"}
- {:name "CREATED_AT", :display_name "Created At", :type "datetime", :semantic_type "creation_timestamp", :table_reference "User"}
- {:name "ID", :display_name "ID", :type "number", :semantic_type "pk", :table_reference "Product"}
- {:name "EAN", :display_name "Ean", :type "string", :field_values string-sequence?, :table_reference "Product"}
- {:name "TITLE", :display_name "Title"
- :type "string"
- :semantic_type "title"
- :field_values string-sequence?
- :table_reference "Product"}
- {:name "CATEGORY", :display_name "Category"
- :type "string"
- :semantic_type "category"
- :field_values string-sequence?
- :table_reference "Product"}
- {:name "VENDOR", :display_name "Vendor"
- :type "string"
- :semantic_type "company"
- :field_values string-sequence?
- :table_reference "Product"}
- {:name "PRICE", :display_name "Price", :type "number", :table_reference "Product"}
- {:name "RATING", :display_name "Rating", :type "number", :semantic_type "score", :table_reference "Product"}
- {:name "CREATED_AT", :display_name "Created At", :type "datetime", :semantic_type "creation_timestamp", :table_reference "Product"}]]
+ {:name "QUANTITY", :display_name "Quantity", :type "number", :semantic_type "quantity", :field_values int-sequence?}]
+ ;; Related tables via FK (order matches what related-tables function returns)
+ expected-related-tables
+ [{:type "table"
+ :id (mt/id :products)
+ :name "PRODUCTS"
+ :display_name "Products"
+ :database_id (mt/id)
+ :database_schema "PUBLIC"
+ :fields
+ [{:name "ID", :display_name "ID", :type "number", :semantic_type "pk"}
+ {:name "EAN", :display_name "Ean", :type "string", :field_values string-sequence?}
+ {:name "TITLE", :display_name "Title", :type "string", :semantic_type "title", :field_values string-sequence?}
+ {:name "CATEGORY", :display_name "Category", :type "string", :semantic_type "category", :field_values string-sequence?}
+ {:name "VENDOR", :display_name "Vendor", :type "string", :semantic_type "company", :field_values string-sequence?}
+ {:name "PRICE", :display_name "Price", :type "number"}
+ {:name "RATING", :display_name "Rating", :type "number", :semantic_type "score"}
+ {:name "CREATED_AT", :display_name "Created At", :type "datetime", :semantic_type "creation_timestamp"}]}
+ {:type "table"
+ :id (mt/id :people)
+ :name "PEOPLE"
+ :display_name "People"
+ :database_id (mt/id)
+ :database_schema "PUBLIC"
+ :fields
+ [{:name "ID", :display_name "ID", :type "number", :semantic_type "pk"}
+ {:name "ADDRESS", :display_name "Address", :type "string"}
+ {:name "EMAIL", :display_name "Email", :type "string", :semantic_type "email"}
+ {:name "PASSWORD", :display_name "Password", :type "string"}
+ {:name "NAME", :display_name "Name", :type "string", :semantic_type "name"}
+ {:name "CITY", :display_name "City", :type "string", :semantic_type "city"}
+ {:name "LONGITUDE", :display_name "Longitude", :type "number", :semantic_type "longitude"}
+ {:name "STATE", :display_name "State", :type "string", :semantic_type "state"}
+ {:name "SOURCE", :display_name "Source", :type "string", :semantic_type "source"}
+ {:name "BIRTH_DATE", :display_name "Birth Date", :type "date"}
+ {:name "ZIP", :display_name "Zip", :type "string"}
+ {:name "LATITUDE", :display_name "Latitude", :type "number", :semantic_type "latitude"}
+ {:name "CREATED_AT", :display_name "Created At", :type "datetime", :semantic_type "creation_timestamp"}]}]]
{:model-data model-data
:metric-data metric-data
- :expected-fields expected-fields}))
+ :expected-core-fields expected-core-fields
+ :expected-related-tables expected-related-tables}))
+
+(defn- add-field-ids
+ "Add field_id to fields using map-indexed. Optionally apply transform-fn to each field."
+ ([prefix-template fields]
+ (add-field-ids prefix-template fields identity))
+ ([prefix-template fields transform-fn]
+ (map-indexed #(transform-fn (assoc %2 :field_id (format prefix-template %1))) fields)))
(deftest get-model-details-basic-test
(mt/with-premium-features #{:metabot-v3}
- (let [{:keys [model-data metric-data expected-fields]} (model-test-fixtures)
+ (let [{:keys [model-data metric-data expected-core-fields expected-related-tables]} (model-test-fixtures)
conversation-id (str (random-uuid))
ai-token (ai-session-token)]
(mt/with-temp [:model/Card {model-id :id} model-data
@@ -799,11 +812,12 @@
(assoc :id model-id
:type "model"
:display_name "Model Model"
- :queryable_foreign_key_tables []
:verified true
- :fields
- (map-indexed #(assoc %2 :field_id (format "c%d/%d" model-id %1))
- expected-fields)
+ :fields (add-field-ids (format "c%d/%%d" model-id) expected-core-fields)
+ :related_tables
+ (map #(update % :fields
+ (partial add-field-ids (format "t%d/%%d" (:id %))))
+ expected-related-tables)
:metrics [(assoc metric-data
:id metric-id
:type "metric"
@@ -813,7 +827,7 @@
(deftest get-model-details-without-field-values-test
(mt/with-premium-features #{:metabot-v3}
- (let [{:keys [model-data metric-data expected-fields]} (model-test-fixtures)
+ (let [{:keys [model-data metric-data expected-core-fields expected-related-tables]} (model-test-fixtures)
conversation-id (str (random-uuid))
ai-token (ai-session-token)]
(mt/with-temp [:model/Card {model-id :id} model-data
@@ -835,13 +849,16 @@
(assoc :id model-id
:type "model"
:display_name "Model Model"
- :queryable_foreign_key_tables []
:verified true
:fields
- (map-indexed #(assoc %2
- :field_id (format "c%d/%d" model-id %1)
- :field_values missing-value)
- expected-fields)
+ (add-field-ids (format "c%d/%%d" model-id) expected-core-fields
+ #(assoc % :field_values missing-value))
+ :related_tables
+ (map (fn [table]
+ (update table :fields
+ #(add-field-ids (format "t%d/%%d" (:id table)) %
+ (fn [f] (assoc f :field_values missing-value)))))
+ expected-related-tables)
:metrics [(assoc metric-data
:id metric-id
:type "metric"
@@ -851,7 +868,7 @@
(deftest get-model-details-without-fields-test
(mt/with-premium-features #{:metabot-v3}
- (let [{:keys [model-data metric-data]} (model-test-fixtures)
+ (let [{:keys [model-data metric-data expected-related-tables]} (model-test-fixtures)
conversation-id (str (random-uuid))
ai-token (ai-session-token)]
(mt/with-temp [:model/Card {model-id :id} model-data
@@ -873,9 +890,10 @@
(assoc :id model-id
:type "model"
:display_name "Model Model"
- :queryable_foreign_key_tables []
:verified true
:fields []
+ :related_tables
+ (map #(assoc % :fields []) expected-related-tables)
:metrics [(assoc metric-data
:id metric-id
:type "metric"
@@ -885,7 +903,7 @@
(deftest get-model-details-without-metric-temporal-breakout-test
(mt/with-premium-features #{:metabot-v3}
- (let [{:keys [model-data metric-data]} (model-test-fixtures)
+ (let [{:keys [model-data metric-data expected-related-tables]} (model-test-fixtures)
conversation-id (str (random-uuid))
ai-token (ai-session-token)]
(mt/with-temp [:model/Card {model-id :id} model-data
@@ -907,9 +925,10 @@
(assoc :id model-id
:type "model"
:display_name "Model Model"
- :queryable_foreign_key_tables []
:verified true
:fields []
+ :related_tables
+ (map #(assoc % :fields []) expected-related-tables)
:metrics [(assoc metric-data
:id metric-id
:type "metric"
@@ -921,7 +940,7 @@
(deftest get-model-details-without-metrics-test
(mt/with-premium-features #{:metabot-v3}
- (let [{:keys [model-data]} (model-test-fixtures)
+ (let [{:keys [model-data expected-related-tables]} (model-test-fixtures)
conversation-id (str (random-uuid))
ai-token (ai-session-token)]
(mt/with-temp [:model/Card {model-id :id} model-data
@@ -943,15 +962,49 @@
(assoc :id model-id
:type "model"
:display_name "Model Model"
- :queryable_foreign_key_tables []
:verified true
:fields []
+ :related_tables
+ (map #(assoc % :fields []) expected-related-tables)
:metrics missing-value))
:conversation_id conversation-id}
(request (assoc arguments
:with_fields false
:with_metrics false)))))))))))
+(deftest get-model-details-without-related-tables-test
+ (mt/with-premium-features #{:metabot-v3}
+ (let [{:keys [model-data]} (model-test-fixtures)
+ conversation-id (str (random-uuid))
+ ai-token (ai-session-token)]
+ (mt/with-temp [:model/Card {model-id :id} model-data
+ :model/Card {_metric-id :id} (assoc (:metric-data (model-test-fixtures)) :dataset_query
+ (mt/mbql-query orders
+ {:source-table (str "card__" model-id)
+ :aggregation [[:count]]
+ :breakout [!month.*created_at *quantity]}))]
+ (with-redefs [metabot-v3.tools.dummy-tools/verified-review? (constantly true)]
+ (let [request (fn [arguments]
+ (mt/user-http-request :rasta :post 200 "ee/metabot-tools/get-table-details"
+ {:request-options {:headers {"x-metabase-session" ai-token}}}
+ {:arguments arguments
+ :conversation_id conversation-id}))]
+ (testing "Without related tables"
+ (is (=? {:structured_output (-> model-data
+ (select-keys [:name :description :database_id])
+ (assoc :id model-id
+ :type "model"
+ :display_name "Model Model"
+ :verified true
+ :fields []
+ :related_tables missing-value
+ :metrics missing-value))
+ :conversation_id conversation-id}
+ (request {:model_id model-id
+ :with_fields false
+ :with_related_tables false
+ :with_metrics false}))))))))))
+
(deftest field-values-auto-populate-test
(mt/with-premium-features #{:metabot-v3}
(t2/delete! :model/FieldValues :field_id [:in (t2/select-fn-vec :id :model/Field :table_id (mt/id :orders))])
@@ -1110,3 +1163,26 @@
(request (assoc arguments
:with_fields false
:with_metrics false))))))))))
+
+(deftest get-table-details-related-tables-test
+ (mt/with-premium-features #{:metabot-v3}
+ (let [conversation-id (str (random-uuid))
+ ai-token (ai-session-token)
+ request (fn [arguments]
+ (mt/user-http-request :rasta :post 200 "ee/metabot-tools/get-table-details"
+ {:request-options {:headers {"x-metabase-session" ai-token}}}
+ {:arguments arguments
+ :conversation_id conversation-id}))]
+ (testing "Normal call includes related_tables by default"
+ (let [response (request {:table_id (mt/id :orders)})
+ related-tables (get-in response [:structured_output :related_tables])]
+ (is (= [(mt/id :products) (mt/id :people)]
+ (map :id related-tables))
+ "Should include tables related to Orders by foreign keys")
+ (is (every? #(not (contains? % :related_tables)) related-tables)
+ "Related tables should not have nested related_tables"))
+
+ (testing "Without related tables"
+ (is (nil? (-> (request {:table_id (mt/id :orders)
+ :with_related_tables false})
+ (get-in [:structured_output :related_tables])))))))))