The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Metabot: two question columns from one field share a field id in read_resource (#86147)
Voytek Pituła7ce42862fe2b
diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.cljindex 59c52d66296..917613d6747 100644--- a/src/metabase/metabot/tools/construct.clj+++ b/src/metabase/metabot/tools/construct.clj@@ -864,7 +864,7 @@[mbql5-query metadata-provider](let [query (lib/query metadata-provider mbql5-query)cols (lib/returned-columns query)]- (mapv #(tools.u/->result-column query %) cols)))+ (tools.u/->result-columns query cols)));;; ---------------------------------------- Query execution ----------------------------------------diff --git a/src/metabase/metabot/tools/entity_details.clj b/src/metabase/metabot/tools/entity_details.cljindex ed30fb08d60..d68b971a98d 100644--- a/src/metabase/metabot/tools/entity_details.clj+++ b/src/metabase/metabot/tools/entity_details.clj@@ -647,7 +647,7 @@(related-tables card-query with-fields? field-values-fn))](-> {:id id:type card-type- :fields (mapv #(metabot.tools.u/->result-column card-query %) returned-fields)+ :fields (metabot.tools.u/->result-columns card-query returned-fields):name (:name base):display_name (some->> (:name base)(u.humanization/name->human-readable-name :simple))@@ -955,7 +955,7 @@{:type :query:query-id query-id:query normalized-query- :result-columns (mapv #(metabot.tools.u/->result-column query %) returned-cols)}))+ :result-columns (metabot.tools.u/->result-columns query returned-cols)}))(defn get-query-details"Get the details of a query (supports both MBQL v4 and v5)."diff --git a/src/metabase/metabot/tools/field_stats.clj b/src/metabase/metabot/tools/field_stats.cljindex 8da084208bc..b25a8d3bd31 100644--- a/src/metabase/metabot/tools/field_stats.clj+++ b/src/metabase/metabot/tools/field_stats.clj@@ -1,6 +1,7 @@(ns metabase.metabot.tools.field-stats(:require[clojure.set :as set]+ [medley.core :as m][metabase.api.common :as api][metabase.lib.core :as lib][metabase.metabot.db :as metabot.db]@@ -100,11 +101,16 @@FK + join label (via `->result-column`) so a drilled-into field/dimension detail tells the LLMexactly how to reference the column."[query col field-id limit]- {:structured-output (merge {:result-type :field-metadata- :field_id field-id- :value_metadata (field-statistics query col limit)}- (select-keys (metabot.tools.u/->result-column query col)- [:portable_fk :table_reference]))})+ (let [;; `field-id` is the field ID, maybe as a string, or a column alias. A column with a field ID that is found by+ ;; its alias is a later column of one field, see `column-field-ids`.+ column-id (if (= (str field-id) (str (:id col)))+ (:id col)+ field-id)]+ {:structured-output (merge {:result-type :field-metadata+ :field_id field-id+ :value_metadata (field-statistics query col limit)}+ (select-keys (metabot.tools.u/->result-column query col column-id)+ [:portable_fk :table_reference]))}))(def ^:private max-suggested-field-ids 100)@@ -127,10 +133,21 @@"Choose another data source."):uses #{}}]})+(defn- usable-columns+ "Keep the columns of `columns`, a list of columns a query can use, that the current user can see, and of these the+ first column of each field ID. Two columns of such a list with one field ID are paths to one field, not two result+ columns, so a later one gets no alias ID.++ The permission filter runs first, because it decides per FK path: the first path to a field can be hidden while a+ later path is visible."+ [columns]+ (m/distinct-by metabot.tools.u/column-field-id (entity-details/permission-filter-columns columns)))+(defn- field-column+ "Find the column of `field-id` in `columns`. The caller removes the columns that the current user cannot see."[entity-type entity-id field-id columns](or (metabot.tools.u/find-column-by-field-id field-id columns)- (let [available-ids (into [] (comp (map metabot.tools.u/column-field-id) (distinct)) columns)]+ (let [available-ids (into [] (distinct) (metabot.tools.u/column-field-ids columns))](unknown-field! {:field-id field-id :entity-type entity-type :entity-id entity-id:available-ids (vec (take max-suggested-field-ids available-ids)):remaining-count (max 0 (- (count available-ids) max-suggested-field-ids))}))))@@ -150,8 +167,7 @@(defn- table-field-stats[table-id field-id limit](let [query (metabot.tools.u/table-query table-id)- columns (entity-details/permission-filter-columns (lib/visible-columns query))- col (field-column "table" table-id field-id columns)]+ col (field-column "table" table-id field-id (usable-columns (lib/visible-columns query)))](field-metadata-output query col field-id limit)))(defn- card-field-stats@@ -160,20 +176,28 @@;; which is what `construct_notebook_query` lets a curated card cover (see;; `metabase.metabot.curation`). The definition of a question reads more: a verified `count` over a raw;; table would otherwise serve every column of that table.- query (if (shared/curated-only?)- (metabot.tools.u/card-results-query card-id (keyword entity-type))- (metabot.tools.u/card-query card-id (keyword entity-type)))- columns (if (= entity-type "question")- (lib/returned-columns query)- (distinct (concat (lib/returned-columns query) (lib/visible-columns query))))- col (field-column entity-type card-id field-id (entity-details/permission-filter-columns columns))]+ query (if (shared/curated-only?)+ (metabot.tools.u/card-results-query card-id (keyword entity-type))+ (metabot.tools.u/card-query card-id (keyword entity-type)))+ returned (entity-details/permission-filter-columns (lib/returned-columns query))+ ;; Result columns keep their own field IDs, see `column-field-ids`. A visible column adds a field ID only+ ;; when no result column has it.+ columns (if (= entity-type "question")+ returned+ (let [returned-ids (set (metabot.tools.u/column-field-ids returned))]+ (concat returned+ (->> (usable-columns (lib/visible-columns query))+ (remove (comp returned-ids metabot.tools.u/column-field-id))))))+ col (field-column entity-type card-id field-id columns)](check-curated-field-owner! card-id col)(field-metadata-output query col field-id limit)))(defn- metric-field-stats[metric-id field-id limit](let [query (metabot.tools.u/metric-query metric-id)- col (field-column "metric" metric-id field-id (entity-details/metric-dimension-columns query metric-id))]+ col (field-column "metric" metric-id field-id+ (m/distinct-by metabot.tools.u/column-field-id+ (entity-details/metric-dimension-columns query metric-id)))](check-curated-field-owner! metric-id col)(field-metadata-output query col field-id limit)))diff --git a/src/metabase/metabot/tools/util.clj b/src/metabase/metabot/tools/util.cljindex 017d19fc697..6c840dd8427 100644--- a/src/metabase/metabot/tools/util.clj+++ b/src/metabase/metabot/tools/util.clj@@ -83,12 +83,33 @@(when-let [tgt (lib.metadata/field query tgt-id)](column-portable-fk query tgt))))+(defn- column-alias+ [column]+ (or (:lib/desired-column-alias column)+ (:lib/source-column-alias column)))+(defn column-field-id"Return the physical field ID or column alias used as a column's `field_id`."[column](or (:id column)- (:lib/desired-column-alias column)- (:lib/source-column-alias column)))+ (column-alias column)))++(defn column-field-ids+ "Return the `field_id` of each column of `columns`, in order. The ids are unique in the list when the columns have+ aliases.++ Two columns can come from one physical field, for example a breakout of `CREATED_AT` by month and by year. The+ first of these columns keeps the field ID. Each later one gets its column alias, as a virtual column does."+ [columns]+ (first+ (reduce (fn [[ids seen] column]+ (let [id (column-field-id column)+ id (if (seen id)+ (or (column-alias column) id)+ id)]+ [(conj ids id) (conj seen id)]))+ [[] #{}]+ columns)))(defn ->result-column"Return tool result column for `column` of `query`.@@ -98,46 +119,63 @@Also includes a `:portable_fk` key with the portable foreign-key path`[db-name, schema, table-name, field-name …]` when the column maps to a real database field.The representations-format notebook query tool expects fields to be referenced by this path- rather than by numeric id."- [query column]- (let [base-type (some-> (:base-type column) u/qualified-name)- effective-type (some-> (:effective-type column) u/qualified-name)- semantic-type (some-> (:semantic-type column) u/qualified-name)- coercion-strategy (some-> (:coercion-strategy column) u/qualified-name)- field-id (column-field-id column)- portable-fk (try (column-portable-fk query column)- (catch Exception _ nil))- fk-target-fk (try (fk-target-portable-fk query column)- (catch Exception _ nil))]- (-> {:field_id field-id- :name (or (:lib/desired-column-alias column)- (:lib/source-column-alias column))- :display_name (lib/display-name query column)- :type (convert-field-type column)}- (m/assoc-some :description (:description column)- :base_type base-type- :effective_type (when (not= effective-type base-type) effective-type)- :semantic_type semantic-type- :database_type (:database-type column)- :coercion_strategy coercion-strategy- :field_values (:field-values column)- :portable_fk portable-fk- :fk_target_portable_fk fk-target-fk- :table_reference (table-reference query column)))))+ rather than by numeric id.++ The 3-arity uses `field-id` as the `field_id`, see [[->result-columns]]. When `field-id` is the alias of a later+ column of one physical field, the column gets no `:portable_fk`: the path names the field, and on a card stage the+ field resolves to the first column."+ ([query column]+ (->result-column query column (column-field-id column)))+ ([query column field-id]+ (let [base-type (some-> (:base-type column) u/qualified-name)+ effective-type (some-> (:effective-type column) u/qualified-name)+ semantic-type (some-> (:semantic-type column) u/qualified-name)+ coercion-strategy (some-> (:coercion-strategy column) u/qualified-name)+ portable-fk (when (= field-id (column-field-id column))+ (try (column-portable-fk query column)+ (catch Exception _ nil)))+ fk-target-fk (try (fk-target-portable-fk query column)+ (catch Exception _ nil))]+ (-> {:field_id field-id+ :name (column-alias column)+ :display_name (lib/display-name query column)+ :type (convert-field-type column)}+ (m/assoc-some :description (:description column)+ :base_type base-type+ :effective_type (when (not= effective-type base-type) effective-type)+ :semantic_type semantic-type+ :database_type (:database-type column)+ :coercion_strategy coercion-strategy+ :field_values (:field-values column)+ :portable_fk portable-fk+ :fk_target_portable_fk fk-target-fk+ :table_reference (table-reference query column))))))++(defn ->result-columns+ "Return tool result columns for `columns` of `query`, with a unique `field_id` for each column. See+ [[column-field-ids]]."+ [query columns]+ (mapv #(->result-column query %1 %2) columns (column-field-ids columns)))(defn find-column-by-field-id"Find a column by its emitted field ID, returning nil if no column matches.- Accepts physical field IDs as integers or numeric strings, and aliases for virtual columns."+ Accepts physical field IDs as integers or numeric strings, and aliases for virtual columns and for later columns of+ one physical field. Matches the ids of [[column-field-ids]]."[field-id columns](when (some? field-id)(let [numeric-id (cond(int? field-id) field-id- (string? field-id) (parse-long field-id))]- (or (m/find-first #(= (column-field-id %) field-id) columns)+ (string? field-id) (parse-long field-id))+ id-columns (map vector (column-field-ids columns) columns)+ find-by-id (fn [id]+ (some (fn [[column-id column]]+ (when (= column-id id) column))+ id-columns))]+ (or (find-by-id field-id)(when numeric-id(m/find-first #(= (:id %) numeric-id) columns))- (m/find-first #(= (column-field-id %) (str field-id)) columns)))))+ (find-by-id (str field-id))))))(defn schedule->schedule-map"Convert a tool schedule map to the schedule-map format used by cron and pulse channels.diff --git a/test/metabase/metabot/tools/field_stats_test.clj b/test/metabase/metabot/tools/field_stats_test.cljindex 1273486e7ad..35ccdeaf540 100644--- a/test/metabase/metabot/tools/field_stats_test.clj+++ b/test/metabase/metabot/tools/field_stats_test.clj@@ -5,6 +5,7 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.metabot.metadata-perms :as metabot.perms]+ [metabase.metabot.tools.entity-details :as entity-details][metabase.metabot.tools.error :as tools.error][metabase.metabot.tools.field-stats :as metabot.tools.field-stats][metabase.test :as mt]@@ -194,3 +195,12 @@(testing "a metric that doesn't exist is refused as not found"(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Not found\."(metric-values Integer/MAX_VALUE))))))))++(deftest usable-columns-filters-before-dedup-test+ (testing "Two FK paths to one field: when the first path is hidden, the column of the visible path stays"+ (let [hidden {:id 1 :fk-field-id 10}+ visible {:id 1 :fk-field-id 11}]+ (mt/with-dynamic-fn-redefs [entity-details/permission-filter-columns+ (fn [columns] (remove #(= 10 (:fk-field-id %)) columns))]+ (is (= [visible]+ (#'metabot.tools.field-stats/usable-columns [hidden visible])))))))diff --git a/test/metabase/metabot/tools/resources_test.clj b/test/metabase/metabot/tools/resources_test.cljindex 969a1fab6dd..2248c10658b 100644--- a/test/metabase/metabot/tools/resources_test.clj+++ b/test/metabase/metabot/tools/resources_test.clj@@ -574,6 +574,44 @@(read-uris{:uris ["metabase://table/99999"]}))))))))+(deftest read-question-fields-of-one-physical-field-test+ (testing "Two breakouts of one field get different field IDs, and each one resolves to its own column"+ (mt/with-current-user (mt/user->id :crowberto)+ (mt/with-temp [:model/Card {card-id :id} {:type :question+ :dataset_query (let [mp (mt/metadata-provider)+ created-at (lib.metadata/field mp (mt/id :orders :created_at))]+ (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/aggregate (lib/count))+ (lib/breakout (lib/with-temporal-bucket created-at :month))+ (lib/breakout (lib/with-temporal-bucket created-at :year))))}]+ (let [created-at-id (mt/id :orders :created_at)+ {:keys [resources]} (read-uris {:uris [(str "metabase://question/" card-id "/fields")]})+ fields (get-in resources [0 :content :structured-output :fields])]+ (is (= [created-at-id "CREATED_AT_2" "count"]+ (map :field_id fields)))+ (testing "a drill-in on each field ID resolves its own column, and only the first column has a portable path"+ ;; Both columns have Field 14, so the output is the same for both. Record the alias of the resolved column.+ (let [output-fn (mt/original-fn #'field-stats/field-metadata-output)+ drill-in (fn [field-id]+ (let [alias (atom nil)]+ (mt/with-dynamic-fn-redefs [field-stats/field-metadata-output+ (fn [query col field-id limit]+ (reset! alias (:lib/desired-column-alias col))+ (output-fn query col field-id limit))]+ (let [{:keys [portable_fk]} (:structured-output+ (field-stats/field-values {:entity-type "question"+ :entity-id card-id+ :field-id field-id+ :limit 5}))]+ {:column-alias @alias :portable-fk? (vector? portable_fk)}))))]+ (are [field-id column-alias portable-fk?] (= {:column-alias column-alias :portable-fk? portable-fk?}+ (drill-in field-id))+ created-at-id "CREATED_AT" true+ "CREATED_AT_2" "CREATED_AT_2" false)))+ (testing "only the first column gets the portable path of the field"+ (is (= [true false false]+ (map (comp vector? :portable_fk) fields)))))))))+(deftest read-dashboard-resource-test(mt/with-current-user (mt/user->id :crowberto)(mt/with-temp [:model/Dashboard {dashboard-id :id dashboard-name :name}diff --git a/test/metabase/metabot/tools/util_test.clj b/test/metabase/metabot/tools/util_test.cljindex 57e41467474..4089c876029 100644--- a/test/metabase/metabot/tools/util_test.clj+++ b/test/metabase/metabot/tools/util_test.clj@@ -306,6 +306,26 @@(are [field-id] (nil? (metabot.tools.util/find-column-by-field-id field-id columns))999 nil "missing")))+(deftest ^:parallel column-field-ids-of-one-physical-field-test+ (testing "Later columns of one physical field get their alias, so each column has its own field ID"+ (let [columns [{:id 14 :lib/desired-column-alias "CREATED_AT"}+ {:id 14 :lib/desired-column-alias "CREATED_AT_2"}+ {:id 14 :lib/source-column-alias "CREATED_AT_3"}+ {:name "count" :lib/desired-column-alias "count"}+ {:id 15}+ {:id 15}]]+ (is (= [14 "CREATED_AT_2" "CREATED_AT_3" "count" 15 15]+ (metabot.tools.util/column-field-ids columns)))+ (testing "and the lookup finds each column by its own field ID"+ (are [field-id col-index] (= (get columns col-index)+ (metabot.tools.util/find-column-by-field-id field-id columns))+ 14 0+ "14" 0+ "CREATED_AT_2" 1+ "CREATED_AT_3" 2+ "count" 3+ 15 4)))))+;;; ---------------------------------------- metric sources ----------------------------------------(deftest ^:parallel metric-required-source-classifies-definition-shapes-test