The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Split fields returned to metabot for cards/models into core fields and related_tables (#64385)
Noah Moss4215b09ac7fb
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.cljindex 872887d4063..1ef36343f10 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.clj@@ -126,17 +126,20 @@(metric-details metadata-provider (assoc options :with-queryable-dimensions? false))(select-keys [:id :type :name :description :default_time_dimension_field_id]))))+(declare related-tables)+(defn- table-details([id] (table-details id nil))- ([id {:keys [metadata-provider field-values-fn with-fields? with-metrics?]- :or {field-values-fn add-field-values- with-fields? true- with-metrics? true}+ ([id {:keys [metadata-provider field-values-fn with-fields? with-related-tables? with-metrics?]+ :or {field-values-fn add-field-values+ with-fields? true+ with-related-tables? true+ with-metrics? true}:as options}](when-let [base (if metadata-provider(lib.metadata/table metadata-provider id)(metabot-v3.tools.u/get-table id :db_id :description :name :schema))]- (let [query-needed? (or with-fields? with-metrics?)+ (let [query-needed? (or with-fields? with-related-tables? with-metrics?)db-id (if metadata-provider (:db-id base) (:db_id base))mp (when query-needed?(or metadata-provider@@ -148,7 +151,9 @@field-values-fn(map #(metabot-v3.tools.u/add-table-reference table-query %))))field-id-prefix (when with-fields?- (metabot-v3.tools.u/table-field-id-prefix id))]+ (metabot-v3.tools.u/table-field-id-prefix id))+ related-tables (when with-related-tables?+ (related-tables table-query with-fields? field-values-fn))](-> {:id id:type :table:fields (into [] (map-indexed #(metabot-v3.tools.u/->result-column table-query %2 %1 field-id-prefix)) cols)@@ -159,27 +164,43 @@:database_id db-id:database_schema (:schema base)}(m/assoc-some :description (:description base)+ :related_tables related-tables:metrics (when with-metrics?(not-empty (mapv #(convert-metric % mp options)(lib/available-metrics table-query))))))))))+(defn- related-tables+ "Constructs a list of tables, optionally including their fields, that are related to the given query via foreign key."+ [query with-fields? field-values-fn]+ (let [fk-fields (->> (lib/visible-columns query)+ (filter :fk-field-id))+ related-table-ids (->> fk-fields+ (map :table-id)+ (into #{}))]+ (when (seq related-table-ids)+ (map #(table-details % {:with-fields? with-fields?+ :field-values-fn field-values-fn+ ;; Important! Only recurse one level+ :with-related-tables? false+ :with-metrics? false})+ related-table-ids))))+(defn- card-details"Get details for a card."([id] (card-details id nil))([id options](when-let [card (metabot-v3.tools.u/get-card id)](card-details card (lib.metadata.jvm/application-database-metadata-provider (:database_id card)) options)))- ([base metadata-provider {:keys [field-values-fn with-fields? with-metrics?]- :or {field-values-fn add-field-values- with-fields? true- with-metrics? true}+ ([base metadata-provider {:keys [field-values-fn with-fields? with-related-tables? with-metrics?]+ :or {field-values-fn add-field-values+ with-fields? true+ with-related-tables? true+ with-metrics? true}:as options}](let [id (:id base)- query-needed? (or with-fields? with-metrics?)- card-metadata (when query-needed?- (lib.metadata/card metadata-provider id))- dataset-query (when query-needed?- (get card-metadata :dataset-query))+ card-metadata (lib.metadata/card metadata-provider id)+ dataset-query (get card-metadata :dataset-query)+ query-needed? (or with-fields? with-related-tables? with-metrics?);; pivot questions have strange result-columns so we work with the dataset-querycard-type (:type base)card-query (when query-needed?@@ -188,25 +209,26 @@(#{:query} (:type dataset-query)))dataset-querycard-metadata)))- cols (when with-fields?- (->> (lib/visible-columns card-query)- field-values-fn- (map #(metabot-v3.tools.u/add-table-reference card-query %))))+ returned-fields (when with-fields?+ (->> (lib/returned-columns card-query)+ field-values-fn))+ related-tables (when with-related-tables?+ (related-tables card-query with-fields? field-values-fn))field-id-prefix (metabot-v3.tools.u/card-field-id-prefix id)](-> {:id id:type card-type- :fields (into [] (map-indexed #(metabot-v3.tools.u/->result-column card-query %2 %1 field-id-prefix)) cols)+ :fields (into [] (map-indexed #(metabot-v3.tools.u/->result-column card-query %2 %1 field-id-prefix)) returned-fields):name (:name base):display_name (some->> (:name base)(u.humanization/name->human-readable-name :simple)):database_id (:database_id base)- :queryable-foreign-key-tables []:verified (verified-review? id "card")}-- (m/assoc-some :description (:description base)- :metrics (when with-metrics?- (not-empty (mapv #(convert-metric % metadata-provider options)- (lib/available-metrics card-query)))))))))+ (m/assoc-some+ :description (:description base)+ :related_tables related-tables+ :metrics (when with-metrics?+ (not-empty (mapv #(convert-metric % metadata-provider options)+ (lib/available-metrics card-query)))))))))(defn cards-details"Get the details of metrics or models as specified by `card-type` and `cards`diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.cljindex eb20f4fd73d..1e045d86cc9 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/api.clj@@ -583,6 +583,7 @@[:table_id {:optional true} [:or :int :string]][:with_fields {:optional true, :default true} :boolean][:with_field_values {:optional true, :default true} :boolean]+ [:with_related_tables {:optional true, :default true} :boolean][:with_metrics {:optional true, :default true} :boolean][:with_metric_default_temporal_breakout {:optional true, :default true} :boolean]][:fn {:error/message "Exactly one of model_id and table_id required"}@@ -592,31 +593,30 @@:table_id :table-id:with_fields :with-fields?:with_field_values :with-field-values?+ :with_related_tables :with-related-tables?:with_metrics :with-metrics?:with_metric_default_temporal_breakout :with-default-temporal-breakout?})}]])-(mr/def ::basic-table- [:map- [:id :int]- [:type [:enum :model :table]]- [:name :string]- [:display_name :string]- [:database_id :int]- [:database_schema {:optional true} [:maybe :string]] ; Schema name, if applicable- [:fields ::columns]- [:description {:optional true} [:maybe :string]]- [:metrics {:optional true} [:sequential ::basic-metric]]])--(mr/def ::full-table- [:merge- ::basic-table- [:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}- [:queryable_foreign_key_tables {:optional true} [:sequential ::basic-table]]]])+(mr/def ::table-result+ [:schema+ {:registry {::table-result+ [:map+ [:id :int]+ [:type [:enum :model :table]]+ [:name :string]+ [:display_name :string]+ [:database_id :int]+ [:database_schema {:optional true} [:maybe :string]] ; Schema name, if applicable+ [:fields ::columns]+ [:related_tables {:optional true} [:sequential [:ref ::table-result]]]+ [:description {:optional true} [:maybe :string]]+ [:metrics {:optional true} [:sequential ::basic-metric]]]}}+ ::table-result])(mr/def ::get-table-details-result[:or[:map {:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}- [:structured_output ::full-table]]+ [:structured_output ::table-result]][:map [:output :string]]])(mr/def ::answer-sources-result@@ -625,7 +625,7 @@{:decode/tool-api-response #(update-keys % metabot-v3.u/safe->snake_case_en)}[:structured_output [:map[:metrics [:sequential ::full-metric]]- [:models [:sequential ::full-table]]]]]+ [:models [:sequential ::table-result]]]]][:map [:output :string]]])(api.macros/defendpoint :post "/answer-sources" :- [:merge ::answer-sources-result ::tool-request]diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.cljindex 352a83578f0..587f1c271c3 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj@@ -727,7 +727,8 @@metric-data {:name "Metric":description "Model based metric":type :metric}- expected-fields+ ;; Core fields returned by the model (lib/returned-columns)+ expected-core-fields[{:name "ID", :display_name "ID", :type "number", :semantic_type "pk"}{:name "USER_ID", :display_name "User ID", :type "number", :semantic_type "fk"}{:name "PRODUCT_ID", :display_name "Product ID", :type "number", :semantic_type "fk"}@@ -736,47 +737,59 @@{:name "TOTAL", :display_name "Total", :type "number"}{:name "DISCOUNT", :display_name "Discount", :type "number", :semantic_type "discount"}{:name "CREATED_AT", :display_name "Created At", :type "datetime", :semantic_type "creation_timestamp"}- {:name "QUANTITY", :display_name "Quantity", :type "number", :semantic_type "quantity", :field_values int-sequence?}- {:name "ID", :display_name "ID", :type "number", :semantic_type "pk", :table_reference "User"}- {:name "ADDRESS", :display_name "Address", :type "string", :table_reference "User"}- {:name "EMAIL", :display_name "Email", :type "string", :semantic_type "email", :table_reference "User"}- {:name "PASSWORD", :display_name "Password", :type "string", :table_reference "User"}- {:name "NAME", :display_name "Name", :type "string", :semantic_type "name", :table_reference "User"}- {:name "CITY", :display_name "City", :type "string", :semantic_type "city", :table_reference "User"}- {:name "LONGITUDE", :display_name "Longitude", :type "number", :semantic_type "longitude", :table_reference "User"}- {:name "STATE", :display_name "State", :type "string", :semantic_type "state", :table_reference "User"}- {:name "SOURCE", :display_name "Source", :type "string", :semantic_type "source", :table_reference "User"}- {:name "BIRTH_DATE", :display_name "Birth Date", :type "date", :table_reference "User"}- {:name "ZIP", :display_name "Zip", :type "string", :table_reference "User"}- {:name "LATITUDE", :display_name "Latitude", :type "number", :semantic_type "latitude", :table_reference "User"}- {:name "CREATED_AT", :display_name "Created At", :type "datetime", :semantic_type "creation_timestamp", :table_reference "User"}- {:name "ID", :display_name "ID", :type "number", :semantic_type "pk", :table_reference "Product"}- {:name "EAN", :display_name "Ean", :type "string", :field_values string-sequence?, :table_reference "Product"}- {:name "TITLE", :display_name "Title"- :type "string"- :semantic_type "title"- :field_values string-sequence?- :table_reference "Product"}- {:name "CATEGORY", :display_name "Category"- :type "string"- :semantic_type "category"- :field_values string-sequence?- :table_reference "Product"}- {:name "VENDOR", :display_name "Vendor"- :type "string"- :semantic_type "company"- :field_values string-sequence?- :table_reference "Product"}- {:name "PRICE", :display_name "Price", :type "number", :table_reference "Product"}- {:name "RATING", :display_name "Rating", :type "number", :semantic_type "score", :table_reference "Product"}- {:name "CREATED_AT", :display_name "Created At", :type "datetime", :semantic_type "creation_timestamp", :table_reference "Product"}]]+ {:name "QUANTITY", :display_name "Quantity", :type "number", :semantic_type "quantity", :field_values int-sequence?}]+ ;; Related tables via FK (order matches what related-tables function returns)+ expected-related-tables+ [{:type "table"+ :id (mt/id :products)+ :name "PRODUCTS"+ :display_name "Products"+ :database_id (mt/id)+ :database_schema "PUBLIC"+ :fields+ [{:name "ID", :display_name "ID", :type "number", :semantic_type "pk"}+ {:name "EAN", :display_name "Ean", :type "string", :field_values string-sequence?}+ {:name "TITLE", :display_name "Title", :type "string", :semantic_type "title", :field_values string-sequence?}+ {:name "CATEGORY", :display_name "Category", :type "string", :semantic_type "category", :field_values string-sequence?}+ {:name "VENDOR", :display_name "Vendor", :type "string", :semantic_type "company", :field_values string-sequence?}+ {:name "PRICE", :display_name "Price", :type "number"}+ {:name "RATING", :display_name "Rating", :type "number", :semantic_type "score"}+ {:name "CREATED_AT", :display_name "Created At", :type "datetime", :semantic_type "creation_timestamp"}]}+ {:type "table"+ :id (mt/id :people)+ :name "PEOPLE"+ :display_name "People"+ :database_id (mt/id)+ :database_schema "PUBLIC"+ :fields+ [{:name "ID", :display_name "ID", :type "number", :semantic_type "pk"}+ {:name "ADDRESS", :display_name "Address", :type "string"}+ {:name "EMAIL", :display_name "Email", :type "string", :semantic_type "email"}+ {:name "PASSWORD", :display_name "Password", :type "string"}+ {:name "NAME", :display_name "Name", :type "string", :semantic_type "name"}+ {:name "CITY", :display_name "City", :type "string", :semantic_type "city"}+ {:name "LONGITUDE", :display_name "Longitude", :type "number", :semantic_type "longitude"}+ {:name "STATE", :display_name "State", :type "string", :semantic_type "state"}+ {:name "SOURCE", :display_name "Source", :type "string", :semantic_type "source"}+ {:name "BIRTH_DATE", :display_name "Birth Date", :type "date"}+ {:name "ZIP", :display_name "Zip", :type "string"}+ {:name "LATITUDE", :display_name "Latitude", :type "number", :semantic_type "latitude"}+ {:name "CREATED_AT", :display_name "Created At", :type "datetime", :semantic_type "creation_timestamp"}]}]]{:model-data model-data:metric-data metric-data- :expected-fields expected-fields}))+ :expected-core-fields expected-core-fields+ :expected-related-tables expected-related-tables}))++(defn- add-field-ids+ "Add field_id to fields using map-indexed. Optionally apply transform-fn to each field."+ ([prefix-template fields]+ (add-field-ids prefix-template fields identity))+ ([prefix-template fields transform-fn]+ (map-indexed #(transform-fn (assoc %2 :field_id (format prefix-template %1))) fields)))(deftest get-model-details-basic-test(mt/with-premium-features #{:metabot-v3}- (let [{:keys [model-data metric-data expected-fields]} (model-test-fixtures)+ (let [{:keys [model-data metric-data expected-core-fields expected-related-tables]} (model-test-fixtures)conversation-id (str (random-uuid))ai-token (ai-session-token)](mt/with-temp [:model/Card {model-id :id} model-data@@ -799,11 +812,12 @@(assoc :id model-id:type "model":display_name "Model Model"- :queryable_foreign_key_tables []:verified true- :fields- (map-indexed #(assoc %2 :field_id (format "c%d/%d" model-id %1))- expected-fields)+ :fields (add-field-ids (format "c%d/%%d" model-id) expected-core-fields)+ :related_tables+ (map #(update % :fields+ (partial add-field-ids (format "t%d/%%d" (:id %))))+ expected-related-tables):metrics [(assoc metric-data:id metric-id:type "metric"@@ -813,7 +827,7 @@(deftest get-model-details-without-field-values-test(mt/with-premium-features #{:metabot-v3}- (let [{:keys [model-data metric-data expected-fields]} (model-test-fixtures)+ (let [{:keys [model-data metric-data expected-core-fields expected-related-tables]} (model-test-fixtures)conversation-id (str (random-uuid))ai-token (ai-session-token)](mt/with-temp [:model/Card {model-id :id} model-data@@ -835,13 +849,16 @@(assoc :id model-id:type "model":display_name "Model Model"- :queryable_foreign_key_tables []:verified true:fields- (map-indexed #(assoc %2- :field_id (format "c%d/%d" model-id %1)- :field_values missing-value)- expected-fields)+ (add-field-ids (format "c%d/%%d" model-id) expected-core-fields+ #(assoc % :field_values missing-value))+ :related_tables+ (map (fn [table]+ (update table :fields+ #(add-field-ids (format "t%d/%%d" (:id table)) %+ (fn [f] (assoc f :field_values missing-value)))))+ expected-related-tables):metrics [(assoc metric-data:id metric-id:type "metric"@@ -851,7 +868,7 @@(deftest get-model-details-without-fields-test(mt/with-premium-features #{:metabot-v3}- (let [{:keys [model-data metric-data]} (model-test-fixtures)+ (let [{:keys [model-data metric-data expected-related-tables]} (model-test-fixtures)conversation-id (str (random-uuid))ai-token (ai-session-token)](mt/with-temp [:model/Card {model-id :id} model-data@@ -873,9 +890,10 @@(assoc :id model-id:type "model":display_name "Model Model"- :queryable_foreign_key_tables []:verified true:fields []+ :related_tables+ (map #(assoc % :fields []) expected-related-tables):metrics [(assoc metric-data:id metric-id:type "metric"@@ -885,7 +903,7 @@(deftest get-model-details-without-metric-temporal-breakout-test(mt/with-premium-features #{:metabot-v3}- (let [{:keys [model-data metric-data]} (model-test-fixtures)+ (let [{:keys [model-data metric-data expected-related-tables]} (model-test-fixtures)conversation-id (str (random-uuid))ai-token (ai-session-token)](mt/with-temp [:model/Card {model-id :id} model-data@@ -907,9 +925,10 @@(assoc :id model-id:type "model":display_name "Model Model"- :queryable_foreign_key_tables []:verified true:fields []+ :related_tables+ (map #(assoc % :fields []) expected-related-tables):metrics [(assoc metric-data:id metric-id:type "metric"@@ -921,7 +940,7 @@(deftest get-model-details-without-metrics-test(mt/with-premium-features #{:metabot-v3}- (let [{:keys [model-data]} (model-test-fixtures)+ (let [{:keys [model-data expected-related-tables]} (model-test-fixtures)conversation-id (str (random-uuid))ai-token (ai-session-token)](mt/with-temp [:model/Card {model-id :id} model-data@@ -943,15 +962,49 @@(assoc :id model-id:type "model":display_name "Model Model"- :queryable_foreign_key_tables []:verified true:fields []+ :related_tables+ (map #(assoc % :fields []) expected-related-tables):metrics missing-value)):conversation_id conversation-id}(request (assoc arguments:with_fields false:with_metrics false)))))))))))+(deftest get-model-details-without-related-tables-test+ (mt/with-premium-features #{:metabot-v3}+ (let [{:keys [model-data]} (model-test-fixtures)+ conversation-id (str (random-uuid))+ ai-token (ai-session-token)]+ (mt/with-temp [:model/Card {model-id :id} model-data+ :model/Card {_metric-id :id} (assoc (:metric-data (model-test-fixtures)) :dataset_query+ (mt/mbql-query orders+ {:source-table (str "card__" model-id)+ :aggregation [[:count]]+ :breakout [!month.*created_at *quantity]}))]+ (with-redefs [metabot-v3.tools.dummy-tools/verified-review? (constantly true)]+ (let [request (fn [arguments]+ (mt/user-http-request :rasta :post 200 "ee/metabot-tools/get-table-details"+ {:request-options {:headers {"x-metabase-session" ai-token}}}+ {:arguments arguments+ :conversation_id conversation-id}))]+ (testing "Without related tables"+ (is (=? {:structured_output (-> model-data+ (select-keys [:name :description :database_id])+ (assoc :id model-id+ :type "model"+ :display_name "Model Model"+ :verified true+ :fields []+ :related_tables missing-value+ :metrics missing-value))+ :conversation_id conversation-id}+ (request {:model_id model-id+ :with_fields false+ :with_related_tables false+ :with_metrics false}))))))))))+(deftest field-values-auto-populate-test(mt/with-premium-features #{:metabot-v3}(t2/delete! :model/FieldValues :field_id [:in (t2/select-fn-vec :id :model/Field :table_id (mt/id :orders))])@@ -1110,3 +1163,26 @@(request (assoc arguments:with_fields false:with_metrics false))))))))))++(deftest get-table-details-related-tables-test+ (mt/with-premium-features #{:metabot-v3}+ (let [conversation-id (str (random-uuid))+ ai-token (ai-session-token)+ request (fn [arguments]+ (mt/user-http-request :rasta :post 200 "ee/metabot-tools/get-table-details"+ {:request-options {:headers {"x-metabase-session" ai-token}}}+ {:arguments arguments+ :conversation_id conversation-id}))]+ (testing "Normal call includes related_tables by default"+ (let [response (request {:table_id (mt/id :orders)})+ related-tables (get-in response [:structured_output :related_tables])]+ (is (= [(mt/id :products) (mt/id :people)]+ (map :id related-tables))+ "Should include tables related to Orders by foreign keys")+ (is (every? #(not (contains? % :related_tables)) related-tables)+ "Related tables should not have nested related_tables"))++ (testing "Without related tables"+ (is (nil? (-> (request {:table_id (mt/id :orders)+ :with_related_tables false})+ (get-in [:structured_output :related_tables])))))))))