Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Metabot: two question columns from one field share a field id in read_resource (#86147)

Voytek Pituła7ce42862fe2b
diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.clj
index 59c52d66296..917613d6747 100644
--- a/src/metabase/metabot/tools/construct.clj
+++ b/src/metabase/metabot/tools/construct.clj
@@ -864,7 +864,7 @@
[mbql5-query metadata-provider]
(let [query (lib/query metadata-provider mbql5-query)
cols (lib/returned-columns query)]
- (mapv #(tools.u/->result-column query %) cols)))
+ (tools.u/->result-columns query cols)))
;;; ---------------------------------------- Query execution ----------------------------------------
diff --git a/src/metabase/metabot/tools/entity_details.clj b/src/metabase/metabot/tools/entity_details.clj
index ed30fb08d60..d68b971a98d 100644
--- a/src/metabase/metabot/tools/entity_details.clj
+++ b/src/metabase/metabot/tools/entity_details.clj
@@ -647,7 +647,7 @@
(related-tables card-query with-fields? field-values-fn))]
(-> {:id id
:type card-type
- :fields (mapv #(metabot.tools.u/->result-column card-query %) returned-fields)
+ :fields (metabot.tools.u/->result-columns card-query returned-fields)
:name (:name base)
:display_name (some->> (:name base)
(u.humanization/name->human-readable-name :simple))
@@ -955,7 +955,7 @@
{:type :query
:query-id query-id
:query normalized-query
- :result-columns (mapv #(metabot.tools.u/->result-column query %) returned-cols)}))
+ :result-columns (metabot.tools.u/->result-columns query returned-cols)}))
(defn get-query-details
"Get the details of a query (supports both MBQL v4 and v5)."
diff --git a/src/metabase/metabot/tools/field_stats.clj b/src/metabase/metabot/tools/field_stats.clj
index 8da084208bc..b25a8d3bd31 100644
--- a/src/metabase/metabot/tools/field_stats.clj
+++ b/src/metabase/metabot/tools/field_stats.clj
@@ -1,6 +1,7 @@
(ns metabase.metabot.tools.field-stats
(:require
[clojure.set :as set]
+ [medley.core :as m]
[metabase.api.common :as api]
[metabase.lib.core :as lib]
[metabase.metabot.db :as metabot.db]
@@ -100,11 +101,16 @@
FK + join label (via `->result-column`) so a drilled-into field/dimension detail tells the LLM
exactly how to reference the column."
[query col field-id limit]
- {:structured-output (merge {:result-type :field-metadata
- :field_id field-id
- :value_metadata (field-statistics query col limit)}
- (select-keys (metabot.tools.u/->result-column query col)
- [:portable_fk :table_reference]))})
+ (let [;; `field-id` is the field ID, maybe as a string, or a column alias. A column with a field ID that is found by
+ ;; its alias is a later column of one field, see `column-field-ids`.
+ column-id (if (= (str field-id) (str (:id col)))
+ (:id col)
+ field-id)]
+ {:structured-output (merge {:result-type :field-metadata
+ :field_id field-id
+ :value_metadata (field-statistics query col limit)}
+ (select-keys (metabot.tools.u/->result-column query col column-id)
+ [:portable_fk :table_reference]))}))
(def ^:private max-suggested-field-ids 100)
@@ -127,10 +133,21 @@
"Choose another data source.")
:uses #{}}]})
+(defn- usable-columns
+ "Keep the columns of `columns`, a list of columns a query can use, that the current user can see, and of these the
+ first column of each field ID. Two columns of such a list with one field ID are paths to one field, not two result
+ columns, so a later one gets no alias ID.
+
+ The permission filter runs first, because it decides per FK path: the first path to a field can be hidden while a
+ later path is visible."
+ [columns]
+ (m/distinct-by metabot.tools.u/column-field-id (entity-details/permission-filter-columns columns)))
+
(defn- field-column
+ "Find the column of `field-id` in `columns`. The caller removes the columns that the current user cannot see."
[entity-type entity-id field-id columns]
(or (metabot.tools.u/find-column-by-field-id field-id columns)
- (let [available-ids (into [] (comp (map metabot.tools.u/column-field-id) (distinct)) columns)]
+ (let [available-ids (into [] (distinct) (metabot.tools.u/column-field-ids columns))]
(unknown-field! {:field-id field-id :entity-type entity-type :entity-id entity-id
:available-ids (vec (take max-suggested-field-ids available-ids))
:remaining-count (max 0 (- (count available-ids) max-suggested-field-ids))}))))
@@ -150,8 +167,7 @@
(defn- table-field-stats
[table-id field-id limit]
(let [query (metabot.tools.u/table-query table-id)
- columns (entity-details/permission-filter-columns (lib/visible-columns query))
- col (field-column "table" table-id field-id columns)]
+ col (field-column "table" table-id field-id (usable-columns (lib/visible-columns query)))]
(field-metadata-output query col field-id limit)))
(defn- card-field-stats
@@ -160,20 +176,28 @@
;; which is what `construct_notebook_query` lets a curated card cover (see
;; `metabase.metabot.curation`). The definition of a question reads more: a verified `count` over a raw
;; table would otherwise serve every column of that table.
- query (if (shared/curated-only?)
- (metabot.tools.u/card-results-query card-id (keyword entity-type))
- (metabot.tools.u/card-query card-id (keyword entity-type)))
- columns (if (= entity-type "question")
- (lib/returned-columns query)
- (distinct (concat (lib/returned-columns query) (lib/visible-columns query))))
- col (field-column entity-type card-id field-id (entity-details/permission-filter-columns columns))]
+ query (if (shared/curated-only?)
+ (metabot.tools.u/card-results-query card-id (keyword entity-type))
+ (metabot.tools.u/card-query card-id (keyword entity-type)))
+ returned (entity-details/permission-filter-columns (lib/returned-columns query))
+ ;; Result columns keep their own field IDs, see `column-field-ids`. A visible column adds a field ID only
+ ;; when no result column has it.
+ columns (if (= entity-type "question")
+ returned
+ (let [returned-ids (set (metabot.tools.u/column-field-ids returned))]
+ (concat returned
+ (->> (usable-columns (lib/visible-columns query))
+ (remove (comp returned-ids metabot.tools.u/column-field-id))))))
+ col (field-column entity-type card-id field-id columns)]
(check-curated-field-owner! card-id col)
(field-metadata-output query col field-id limit)))
(defn- metric-field-stats
[metric-id field-id limit]
(let [query (metabot.tools.u/metric-query metric-id)
- col (field-column "metric" metric-id field-id (entity-details/metric-dimension-columns query metric-id))]
+ col (field-column "metric" metric-id field-id
+ (m/distinct-by metabot.tools.u/column-field-id
+ (entity-details/metric-dimension-columns query metric-id)))]
(check-curated-field-owner! metric-id col)
(field-metadata-output query col field-id limit)))
diff --git a/src/metabase/metabot/tools/util.clj b/src/metabase/metabot/tools/util.clj
index 017d19fc697..6c840dd8427 100644
--- a/src/metabase/metabot/tools/util.clj
+++ b/src/metabase/metabot/tools/util.clj
@@ -83,12 +83,33 @@
(when-let [tgt (lib.metadata/field query tgt-id)]
(column-portable-fk query tgt))))
+(defn- column-alias
+ [column]
+ (or (:lib/desired-column-alias column)
+ (:lib/source-column-alias column)))
+
(defn column-field-id
"Return the physical field ID or column alias used as a column's `field_id`."
[column]
(or (:id column)
- (:lib/desired-column-alias column)
- (:lib/source-column-alias column)))
+ (column-alias column)))
+
+(defn column-field-ids
+ "Return the `field_id` of each column of `columns`, in order. The ids are unique in the list when the columns have
+ aliases.
+
+ Two columns can come from one physical field, for example a breakout of `CREATED_AT` by month and by year. The
+ first of these columns keeps the field ID. Each later one gets its column alias, as a virtual column does."
+ [columns]
+ (first
+ (reduce (fn [[ids seen] column]
+ (let [id (column-field-id column)
+ id (if (seen id)
+ (or (column-alias column) id)
+ id)]
+ [(conj ids id) (conj seen id)]))
+ [[] #{}]
+ columns)))
(defn ->result-column
"Return tool result column for `column` of `query`.
@@ -98,46 +119,63 @@
Also includes a `:portable_fk` key with the portable foreign-key path
`[db-name, schema, table-name, field-name …]` when the column maps to a real database field.
The representations-format notebook query tool expects fields to be referenced by this path
- rather than by numeric id."
- [query column]
- (let [base-type (some-> (:base-type column) u/qualified-name)
- effective-type (some-> (:effective-type column) u/qualified-name)
- semantic-type (some-> (:semantic-type column) u/qualified-name)
- coercion-strategy (some-> (:coercion-strategy column) u/qualified-name)
- field-id (column-field-id column)
- portable-fk (try (column-portable-fk query column)
- (catch Exception _ nil))
- fk-target-fk (try (fk-target-portable-fk query column)
- (catch Exception _ nil))]
- (-> {:field_id field-id
- :name (or (:lib/desired-column-alias column)
- (:lib/source-column-alias column))
- :display_name (lib/display-name query column)
- :type (convert-field-type column)}
- (m/assoc-some :description (:description column)
- :base_type base-type
- :effective_type (when (not= effective-type base-type) effective-type)
- :semantic_type semantic-type
- :database_type (:database-type column)
- :coercion_strategy coercion-strategy
- :field_values (:field-values column)
- :portable_fk portable-fk
- :fk_target_portable_fk fk-target-fk
- :table_reference (table-reference query column)))))
+ rather than by numeric id.
+
+ The 3-arity uses `field-id` as the `field_id`, see [[->result-columns]]. When `field-id` is the alias of a later
+ column of one physical field, the column gets no `:portable_fk`: the path names the field, and on a card stage the
+ field resolves to the first column."
+ ([query column]
+ (->result-column query column (column-field-id column)))
+ ([query column field-id]
+ (let [base-type (some-> (:base-type column) u/qualified-name)
+ effective-type (some-> (:effective-type column) u/qualified-name)
+ semantic-type (some-> (:semantic-type column) u/qualified-name)
+ coercion-strategy (some-> (:coercion-strategy column) u/qualified-name)
+ portable-fk (when (= field-id (column-field-id column))
+ (try (column-portable-fk query column)
+ (catch Exception _ nil)))
+ fk-target-fk (try (fk-target-portable-fk query column)
+ (catch Exception _ nil))]
+ (-> {:field_id field-id
+ :name (column-alias column)
+ :display_name (lib/display-name query column)
+ :type (convert-field-type column)}
+ (m/assoc-some :description (:description column)
+ :base_type base-type
+ :effective_type (when (not= effective-type base-type) effective-type)
+ :semantic_type semantic-type
+ :database_type (:database-type column)
+ :coercion_strategy coercion-strategy
+ :field_values (:field-values column)
+ :portable_fk portable-fk
+ :fk_target_portable_fk fk-target-fk
+ :table_reference (table-reference query column))))))
+
+(defn ->result-columns
+ "Return tool result columns for `columns` of `query`, with a unique `field_id` for each column. See
+ [[column-field-ids]]."
+ [query columns]
+ (mapv #(->result-column query %1 %2) columns (column-field-ids columns)))
(defn find-column-by-field-id
"Find a column by its emitted field ID, returning nil if no column matches.
- Accepts physical field IDs as integers or numeric strings, and aliases for virtual columns."
+ Accepts physical field IDs as integers or numeric strings, and aliases for virtual columns and for later columns of
+ one physical field. Matches the ids of [[column-field-ids]]."
[field-id columns]
(when (some? field-id)
(let [numeric-id (cond
(int? field-id) field-id
- (string? field-id) (parse-long field-id))]
- (or (m/find-first #(= (column-field-id %) field-id) columns)
+ (string? field-id) (parse-long field-id))
+ id-columns (map vector (column-field-ids columns) columns)
+ find-by-id (fn [id]
+ (some (fn [[column-id column]]
+ (when (= column-id id) column))
+ id-columns))]
+ (or (find-by-id field-id)
(when numeric-id
(m/find-first #(= (:id %) numeric-id) columns))
- (m/find-first #(= (column-field-id %) (str field-id)) columns)))))
+ (find-by-id (str field-id))))))
(defn schedule->schedule-map
"Convert a tool schedule map to the schedule-map format used by cron and pulse channels.
diff --git a/test/metabase/metabot/tools/field_stats_test.clj b/test/metabase/metabot/tools/field_stats_test.clj
index 1273486e7ad..35ccdeaf540 100644
--- a/test/metabase/metabot/tools/field_stats_test.clj
+++ b/test/metabase/metabot/tools/field_stats_test.clj
@@ -5,6 +5,7 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.metabot.metadata-perms :as metabot.perms]
+ [metabase.metabot.tools.entity-details :as entity-details]
[metabase.metabot.tools.error :as tools.error]
[metabase.metabot.tools.field-stats :as metabot.tools.field-stats]
[metabase.test :as mt]
@@ -194,3 +195,12 @@
(testing "a metric that doesn't exist is refused as not found"
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Not found\."
(metric-values Integer/MAX_VALUE))))))))
+
+(deftest usable-columns-filters-before-dedup-test
+ (testing "Two FK paths to one field: when the first path is hidden, the column of the visible path stays"
+ (let [hidden {:id 1 :fk-field-id 10}
+ visible {:id 1 :fk-field-id 11}]
+ (mt/with-dynamic-fn-redefs [entity-details/permission-filter-columns
+ (fn [columns] (remove #(= 10 (:fk-field-id %)) columns))]
+ (is (= [visible]
+ (#'metabot.tools.field-stats/usable-columns [hidden visible])))))))
diff --git a/test/metabase/metabot/tools/resources_test.clj b/test/metabase/metabot/tools/resources_test.clj
index 969a1fab6dd..2248c10658b 100644
--- a/test/metabase/metabot/tools/resources_test.clj
+++ b/test/metabase/metabot/tools/resources_test.clj
@@ -574,6 +574,44 @@
(read-uris
{:uris ["metabase://table/99999"]}))))))))
+(deftest read-question-fields-of-one-physical-field-test
+ (testing "Two breakouts of one field get different field IDs, and each one resolves to its own column"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temp [:model/Card {card-id :id} {:type :question
+ :dataset_query (let [mp (mt/metadata-provider)
+ created-at (lib.metadata/field mp (mt/id :orders :created_at))]
+ (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ (lib/aggregate (lib/count))
+ (lib/breakout (lib/with-temporal-bucket created-at :month))
+ (lib/breakout (lib/with-temporal-bucket created-at :year))))}]
+ (let [created-at-id (mt/id :orders :created_at)
+ {:keys [resources]} (read-uris {:uris [(str "metabase://question/" card-id "/fields")]})
+ fields (get-in resources [0 :content :structured-output :fields])]
+ (is (= [created-at-id "CREATED_AT_2" "count"]
+ (map :field_id fields)))
+ (testing "a drill-in on each field ID resolves its own column, and only the first column has a portable path"
+ ;; Both columns have Field 14, so the output is the same for both. Record the alias of the resolved column.
+ (let [output-fn (mt/original-fn #'field-stats/field-metadata-output)
+ drill-in (fn [field-id]
+ (let [alias (atom nil)]
+ (mt/with-dynamic-fn-redefs [field-stats/field-metadata-output
+ (fn [query col field-id limit]
+ (reset! alias (:lib/desired-column-alias col))
+ (output-fn query col field-id limit))]
+ (let [{:keys [portable_fk]} (:structured-output
+ (field-stats/field-values {:entity-type "question"
+ :entity-id card-id
+ :field-id field-id
+ :limit 5}))]
+ {:column-alias @alias :portable-fk? (vector? portable_fk)}))))]
+ (are [field-id column-alias portable-fk?] (= {:column-alias column-alias :portable-fk? portable-fk?}
+ (drill-in field-id))
+ created-at-id "CREATED_AT" true
+ "CREATED_AT_2" "CREATED_AT_2" false)))
+ (testing "only the first column gets the portable path of the field"
+ (is (= [true false false]
+ (map (comp vector? :portable_fk) fields)))))))))
+
(deftest read-dashboard-resource-test
(mt/with-current-user (mt/user->id :crowberto)
(mt/with-temp [:model/Dashboard {dashboard-id :id dashboard-name :name}
diff --git a/test/metabase/metabot/tools/util_test.clj b/test/metabase/metabot/tools/util_test.clj
index 57e41467474..4089c876029 100644
--- a/test/metabase/metabot/tools/util_test.clj
+++ b/test/metabase/metabot/tools/util_test.clj
@@ -306,6 +306,26 @@
(are [field-id] (nil? (metabot.tools.util/find-column-by-field-id field-id columns))
999 nil "missing")))
+(deftest ^:parallel column-field-ids-of-one-physical-field-test
+ (testing "Later columns of one physical field get their alias, so each column has its own field ID"
+ (let [columns [{:id 14 :lib/desired-column-alias "CREATED_AT"}
+ {:id 14 :lib/desired-column-alias "CREATED_AT_2"}
+ {:id 14 :lib/source-column-alias "CREATED_AT_3"}
+ {:name "count" :lib/desired-column-alias "count"}
+ {:id 15}
+ {:id 15}]]
+ (is (= [14 "CREATED_AT_2" "CREATED_AT_3" "count" 15 15]
+ (metabot.tools.util/column-field-ids columns)))
+ (testing "and the lookup finds each column by its own field ID"
+ (are [field-id col-index] (= (get columns col-index)
+ (metabot.tools.util/find-column-by-field-id field-id columns))
+ 14 0
+ "14" 0
+ "CREATED_AT_2" 1
+ "CREATED_AT_3" 2
+ "count" 3
+ 15 4)))))
+
;;; ---------------------------------------- metric sources ----------------------------------------
(deftest ^:parallel metric-required-source-classifies-definition-shapes-test