The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix metadata enrichment permissions for linked tables (#78896)
Chris Truter2edc737aea66
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 685539ae529..95465725343 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1326,6 +1326,7 @@liblib-bellm+ metricsmodelsnative-query-snippetsnotificationdiff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/entity_details_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/entity_details_test.cljindex a8b18f17bf1..61755e1bf15 100644--- a/enterprise/backend/test/metabase_enterprise/metabot/tools/entity_details_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/entity_details_test.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.metabot.tools.entity-details-test(:require[clojure.test :refer :all]+ [metabase-enterprise.sandbox.test-util :as sandbox.tu][metabase-enterprise.test :as met][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -24,3 +25,33 @@(is (= ["African" "American"] (:field_values name-field)))))(finally(t2/delete! :model/FieldValues :field_id field-id :type :advanced))))))++(deftest sandboxed-metric-dimensions-test+ (testing "metric details respect column-restricting sandboxes"+ (met/with-gtaps! {:gtaps {:venues {:query (sandbox.tu/restricted-column-query (mt/id))}}}+ (mt/with-temp [:model/Card {metric-id :id} {:name "Sandboxed metric"+ :type :metric+ :database_id (mt/id)+ :table_id (mt/id :venues)+ :dataset_query {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :venues)+ :aggregation [[:count]]}}}]+ (let [venue-name (:name (lib.metadata/table (mt/metadata-provider) (mt/id :venues)))+ metric-dimensions (fn []+ (-> (entity-details/get-metric-details {:metric-id metric-id+ :with-field-values? false})+ :structured-output+ :queryable-dimensions))+ source-names (fn [dimensions]+ (into #{}+ (comp (filter #(= venue-name (get-in % [:portable_fk 2])))+ (map :name))+ dimensions))]+ (testing "only source-card columns are returned"+ (is (= #{"CATEGORY_ID" "ID" "NAME"}+ (source-names (metric-dimensions)))))+ (testing "missing sandbox metadata fails closed"+ (let [sandbox (t2/select-one :model/Sandbox :table_id (mt/id :venues))]+ (t2/update! :model/Card :id (:card_id sandbox) {:result_metadata nil})+ (is (empty? (metric-dimensions))))))))))diff --git a/src/metabase/llm/context.clj b/src/metabase/llm/context.cljindex b044ff11022..63788bc2912 100644--- a/src/metabase/llm/context.clj+++ b/src/metabase/llm/context.clj@@ -174,20 +174,21 @@(defn- fetch-fk-targets"Fetch table.field names for FK target fields.+ Only includes targets whose Tables the current user can access.Returns map of target-field-id -> {:table name :field name}"[columns](let [target-ids (->> columns(keep :fk_target_field_id)set)](when (seq target-ids)- (let [fields (t2/select [:model/Field :id :name :table_id]- :id [:in target-ids])- table-ids (into #{} (map :table_id) fields)- table-names (when (seq table-ids)- (t2/select-pk->fn :name :model/Table :id [:in table-ids]))]+ (let [fields (t2/select [:model/Field :id :name :table_id]+ :id [:in target-ids])+ table-ids (into #{} (map :table_id) fields)+ accessible-tables (fetch-accessible-tables table-ids)](into {}- (map (fn [{:keys [id name table_id]}]- [id {:table (get table-names table_id) :field name}]))+ (keep (fn [{:keys [id name table_id]}]+ (when-let [table (get accessible-tables table_id)]+ [id {:table (:name table) :field name}])))fields)))));;; ----------------------------------------- On-Demand Metadata Enrichment -----------------------------------------diff --git a/src/metabase/metabot/tools/entity_details.clj b/src/metabase/metabot/tools/entity_details.cljindex a0ce55fc493..1a04e190773 100644--- a/src/metabase/metabot/tools/entity_details.clj+++ b/src/metabase/metabot/tools/entity_details.clj@@ -11,6 +11,8 @@[metabase.metabot.config :as metabot.config][metabase.metabot.tools.shared.content-store :as shared.content-store][metabase.metabot.tools.util :as metabot.tools.u]+ [metabase.metrics.core :as metrics]+ [metabase.models.interface :as mi][metabase.parameters.field-values :as params.field-values][metabase.util :as u][metabase.util.humanization :as u.humanization]@@ -130,9 +132,12 @@"Get field values for a field, creating them if they don't exist.Uses the user-aware API that respects sandboxing/impersonation."[id->values id]- (-> (get id->values id)- (or (params.field-values/get-or-create-field-values! (t2/select-one :model/Field :id id)))- :values))+ (if-some [field-values (get id->values id)]+ (:values field-values)+ (let [field (t2/select-one :model/Field :id id)]+ (when (and field+ (params.field-values/current-user-can-fetch-field-values? field))+ (:values (params.field-values/get-or-create-field-values! field))))))(defn- add-field-values[cols]@@ -141,6 +146,49 @@(map #(m/assoc-some % :field-values (some->> % :id (get-field-values id->values))) cols))cols))+(defn- permission-filter-columns+ "Remove columns hidden by Table or sandbox permissions and hide inaccessible FK targets on retained columns."+ [columns]+ (let [columns (vec columns)+ referenced-field-ids (into #{}+ (comp (mapcat (juxt :fk-field-id :fk-target-field-id))+ (filter some?))+ columns)+ referenced-fields (when (seq referenced-field-ids)+ (t2/select [:model/Field :id :table_id]+ :id [:in referenced-field-ids]))+ field-id->field (m/index-by :id referenced-fields)+ table-ids (into #{}+ (filter pos-int?)+ (concat (keep :table-id columns)+ (keep :table_id referenced-fields)))+ readable-table-ids (->> (t2/select :model/Table :id [:in table-ids])+ (filter mi/can-read?)+ (map :id)+ set)+ sandbox-restricted-ids (metrics/sandbox-restricted-fields table-ids)+ field-readable? (fn [table-id field]+ (and (contains? readable-table-ids table-id)+ (if-let [allowed-field-ids (get sandbox-restricted-ids table-id)]+ (contains? allowed-field-ids (u/id field))+ true)))+ referenced-field-readable?+ (fn [field-id]+ (when-let [field (get field-id->field field-id)]+ (field-readable? (:table_id field) field)))+ column-readable? (fn [{:keys [fk-field-id table-id] :as column}]+ (and (or (not (pos-int? table-id))+ (field-readable? table-id column))+ (or (nil? fk-field-id)+ (referenced-field-readable? fk-field-id))))]+ (->> columns+ (filter column-readable?)+ (mapv (fn [{:keys [fk-target-field-id] :as column}]+ (cond-> column+ (and fk-target-field-id+ (not (referenced-field-readable? fk-target-field-id)))+ (dissoc :fk-target-field-id)))))))+(defn metric-details"Get metric details as returned by tools."([id] (metric-details id nil))@@ -175,12 +223,16 @@;; We accept both. `report_card.table_id` / metadata `:table-id` already points;; to the metric's base table and is kept in sync, so we use it verbatim instead;; of digging through `:dataset_query`.+ ;; Reading the metric Card is collection-based and does not imply permission to+ ;; reveal metadata for this physical Table.source-table-id (or (:table-id card) (:table_id card))- source-table (when source-table-id+ source-table (when (and source-table-id+ (mi/can-read? :model/Table source-table-id))(lib.metadata/table metadata-provider source-table-id))base-table-portable-fk (when (and database-name source-table)[database-name (:schema source-table) (:name source-table)])- query-needed? (or with-default-temporal-breakout? with-queryable-dimensions? with-segments?)+ query-needed? (and source-table+ (or with-default-temporal-breakout? with-queryable-dimensions? with-segments?))metric-query (when query-needed?(lib/query metadata-provider (lib.metadata/card metadata-provider id)))breakouts (when query-needed?@@ -189,11 +241,16 @@(lib/remove-all-breakouts metric-query))visible-cols (when query-needed?(->> (lib/visible-columns base-query)+ permission-filter-columns(map #(metabot.tools.u/add-table-reference base-query %))))- default-temporal-breakout (when with-default-temporal-breakout?+ default-temporal-breakout (when (and query-needed? with-default-temporal-breakout?)(->> breakouts(map #(lib/find-matching-column % visible-cols))- (m/find-first lib.types.isa/temporal?)))]+ (m/find-first lib.types.isa/temporal?)))+ queryable-columns (when (and query-needed? with-queryable-dimensions?)+ (->> (lib/filterable-columns base-query)+ permission-filter-columns+ field-values-fn))](cond-> {:id id:type :metric:name (:name card)@@ -201,28 +258,32 @@;; Database identity — same shape as in `table-details` / `card-details`.:database_id database-id:database_name database-name- ;; Base table the metric aggregates. The LLM uses `:base_table_portable_fk`- ;; verbatim as `source-table:` in the query that consumes the metric.- :base_table_id source-table-id- :base_table_name (:name source-table)- :base_table_portable_fk base-table-portable-fk;; Portable entity id — the string the LLM must copy verbatim into a;; `[metric, {}, <entity_id>]` aggregation clause to reference this metric.;; Accept both shapes (see `source-table-id` note above): t2 rows use;; `:entity_id`, `lib.metadata/card` maps use `:entity-id`.:portable_entity_id (or (:entity-id card) (:entity_id card))- :default_time_dimension_field_id (some-> default-temporal-breakout- (->> (metabot.tools.u/->result-column metric-query))- :field_id):verified (verified-review? id "card")}- with-queryable-dimensions?+ ;; Base table the metric aggregates. The LLM uses `:base_table_portable_fk`+ ;; verbatim as `source-table:` in the query that consumes the metric. These fields+ ;; are added only when the base Table is readable.+ source-table+ (assoc :base_table_id source-table-id+ :base_table_name (:name source-table)+ :base_table_portable_fk base-table-portable-fk)++ (and source-table with-default-temporal-breakout?)+ (assoc :default_time_dimension_field_id (some-> default-temporal-breakout+ (->> (metabot.tools.u/->result-column metric-query))+ :field_id))++ (and source-table with-queryable-dimensions?)(assoc :queryable-dimensions (into [](comp (map #(metabot.tools.u/add-table-reference base-query %))(map #(metabot.tools.u/->result-column metric-query %)))- (->> (lib/filterable-columns base-query)- field-values-fn)))+ queryable-columns))- with-segments?+ (and source-table with-segments?)(assoc :segments (if-let [segments (lib/available-segments metric-query)](mapv #(convert-measure-or-segment % :filters) segments)[]))))))diff --git a/src/metabase/metabot/tools/search.clj b/src/metabase/metabot/tools/search.cljindex 9dee6113c83..e197ce679ef 100644--- a/src/metabase/metabot/tools/search.clj+++ b/src/metabase/metabot/tools/search.clj@@ -147,7 +147,9 @@hallucinate the base table (observed failure mode: `[<db>, public, customers]`) or do anextra `read_resource` round-trip. We read the two columns directly from`report_card.table_id` + `metabase_table.{schema,name}` to keep the lookup O(1) extra- query per search call, regardless of number of metrics in the result set.+ query per search call, regardless of number of metrics in the result set. Base-table+ metadata is attached only when the current user can read that Table; collection access+ to the metric Card does not imply access to its physical source.Requires `:database_name` to already be set on each metric result (done earlier by[[enrich-with-database-engines]]) so we can assemble the full portable FK@@ -158,21 +160,32 @@(t2/select-pk->fn :table_id :model/Card :id [:in metric-ids]))table-ids (->> card-id->table-id vals (remove nil?) distinct)table-id->info (when (seq table-ids)- (t2/select-pk->fn (juxt :schema :name) :model/Table :id [:in table-ids]))]+ (into {}+ (comp (filter mi/can-read?)+ (map (juxt :id (juxt :schema :name))))+ (t2/select [:model/Table :id :schema :name :db_id]+ :id [:in table-ids])))+ metric-id->table-info+ (into {}+ (keep (fn [[metric-id table-id]]+ (when-let [[schema table-name] (get table-id->info table-id)]+ [metric-id {:table-id table-id+ :schema schema+ :table-name table-name}])))+ card-id->table-id)](cond->> results(seq card-id->table-id)- (mapv (fn [r]- (if (= "metric" (:type r))- (if-let [table-id (get card-id->table-id (:id r))]- (let [[schema table-name] (get table-id->info table-id)- db-name (:database_name r)]- (cond-> (assoc r :base_table_id table-id)- table-name (assoc :base_table_name table-name- :base_table_schema schema)- (and db-name table-name)- (assoc :base_table_portable_fk [db-name schema table-name])))- r)- r))))))+ (mapv (fn [{:keys [id type database_name] :as result}]+ (if-let [{:keys [table-id schema table-name]}+ (when (= "metric" type)+ (get metric-id->table-info id))]+ (cond-> (assoc result+ :base_table_id table-id+ :base_table_name table-name+ :base_table_schema schema)+ database_name+ (assoc :base_table_portable_fk [database_name schema table-name]))+ result))))))(defn- remove-unreadable-transforms"Remove transforms from search results that the user cannot read.diff --git a/src/metabase/metrics/core.clj b/src/metabase/metrics/core.cljindex 1bf5493b23c..1b0241a744f 100644--- a/src/metabase/metrics/core.clj+++ b/src/metabase/metrics/core.clj@@ -19,7 +19,8 @@dimension-search-valuesdimension-remapped-value][metrics.perms- filter-dimensions-for-user]+ filter-dimensions-for-user+ sandbox-restricted-fields][metrics.transformsnormalize-dimensionnormalize-target-refdiff --git a/test/metabase/llm/context_test.clj b/test/metabase/llm/context_test.cljindex 754216c840d..364fb4b3663 100644--- a/test/metabase/llm/context_test.clj+++ b/test/metabase/llm/context_test.clj@@ -3,6 +3,8 @@[clojure.string :as str][clojure.test :refer :all][metabase.llm.context :as context]+ [metabase.permissions.core :as perms]+ [metabase.permissions.models.permissions-group :as perms-group][metabase.test :as mt]))(set! *warn-on-reflection* true)@@ -286,6 +288,40 @@(is (some? result))(is (str/includes? result "FK->users.id")))))))+(deftest get-tables-with-columns-hides-unreadable-fk-target-test+ (testing "an FK on a readable table does not reveal an unreadable target table or field"+ (mt/with-temp [:model/Database db {}+ :model/Table target {:db_id (:id db)+ :name "secret_table"+ :schema "private"}+ :model/Field target-id {:table_id (:id target)+ :name "secret_id"+ :database_type "INTEGER"+ :base_type :type/Integer}+ :model/Table source {:db_id (:id db)+ :name "orders"+ :schema "public"}+ :model/Field _source-id {:table_id (:id source)+ :name "id"+ :database_type "INTEGER"+ :base_type :type/Integer}+ :model/Field _source-fk {:table_id (:id source)+ :name "user_id"+ :database_type "INTEGER"+ :base_type :type/Integer+ :semantic_type :type/FK+ :fk_target_field_id (:id target-id)}]+ (mt/with-no-data-perms-for-all-users!+ (perms/set-database-permission! (perms-group/all-users) db :perms/view-data :unrestricted)+ (perms/set-table-permission! (perms-group/all-users) source :perms/create-queries :query-builder-and-native)+ (mt/with-test-user :rasta+ (let [result (context/get-tables-with-columns (:id db) #{(:id source)})+ fk-col (some #(when (= "user_id" (:name %)) %) (-> result first :columns))]+ (is (some? fk-col) "the readable source FK column is returned")+ (is (not (contains? fk-col :fk_target)))+ (is (not (str/includes? (pr-str result) "secret_table")))+ (is (not (str/includes? (pr-str result) "secret_id")))))))))+(deftest build-schema-context-with-field-values-test(mt/with-test-user :crowberto(mt/with-temp [:model/Database db {}diff --git a/test/metabase/metabot/tools/entity_details_test.clj b/test/metabase/metabot/tools/entity_details_test.cljindex 4ff7c33ae85..4b95b9e9240 100644--- a/test/metabase/metabot/tools/entity_details_test.clj+++ b/test/metabase/metabot/tools/entity_details_test.clj@@ -1,10 +1,15 @@(ns metabase.metabot.tools.entity-details-test{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.metabot.tools.entity-details-test]}}}}}}(:require+ [clojure.string :as str][clojure.test :refer :all]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.metabot.tools.entity-details :as entity-details]+ [metabase.parameters.field-values :as params.field-values]+ [metabase.permissions.core :as perms]+ [metabase.permissions.models.permissions-group :as perms-group][metabase.test :as mt][metabase.test.fixtures :as fixtures][toucan2.core :as t2]))@@ -395,6 +400,20 @@(is (= segment-id (:id segment)))(is (= "Large Orders" (:name segment)))))))))))+(deftest get-field-values-has-stable-shape-test+ (let [field-id (mt/id :categories :name)+ raw-values ["African" "American"]]+ (testing "cache hits return raw values"+ (is (= raw-values+ (#'entity-details/get-field-values {field-id {:values raw-values}} field-id))))+ (testing "cache misses return the same raw-value shape"+ (with-redefs [params.field-values/current-user-can-fetch-field-values? (constantly true)+ params.field-values/get-or-create-field-values! (constantly {:values raw-values})+ params.field-values/get-or-create-field-values-for-current-user!+ (constantly {:values (mapv vector raw-values)})]+ (is (= raw-values+ (#'entity-details/get-field-values {} field-id)))))))+;;; ============================================================;;; Base-table surfacing on get-metric-details (regression);;; ============================================================@@ -426,6 +445,79 @@(:base_table_portable_fk output))"portable FK should be `[database_name, schema, table_name]`")))))))+(deftest get-metric-details-hides-unreadable-base-table-test+ (testing "metric details do not reveal metadata for a base table the user cannot read"+ (let [mp (mt/metadata-provider)+ metric-query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/aggregate (lib/sum (lib.metadata/field mp (mt/id :orders :total)))))]+ (mt/with-temp [:model/Card {metric-id :id} {:dataset_query metric-query+ :database_id (mt/id)+ :name "Restricted base-table metric"+ :type :metric}]+ (mt/with-no-data-perms-for-all-users!+ (mt/with-current-user (mt/user->id :rasta)+ (with-redefs [params.field-values/field-id->field-values-for-current-user+ (fn [_]+ (throw (ex-info "field values must not be fetched" {})))]+ (let [output (:structured-output+ (entity-details/get-metric-details {:metric-id metric-id+ :with-segments? true}))]+ (is (= metric-id (:id output)) "collection access still makes the metric readable")+ (is (not-any? #(contains? output %)+ [:base_table_id+ :base_table_name+ :base_table_portable_fk+ :default_time_dimension_field_id+ :queryable-dimensions+ :segments]))))))))))++(deftest get-metric-details-hides-unreadable-fk-target-test+ (testing "metric dimensions do not reveal metadata for an unreadable FK target table"+ (mt/with-temp [:model/Database db {}+ :model/Table target {:db_id (:id db)+ :name "secret_table"+ :schema "private"}+ :model/Field target-id {:table_id (:id target)+ :name "secret_id"+ :database_type "INTEGER"+ :base_type :type/Integer}+ :model/Table source {:db_id (:id db)+ :name "orders"+ :schema "public"}+ :model/Field _source-id {:table_id (:id source)+ :name "id"+ :database_type "INTEGER"+ :base_type :type/Integer}+ :model/Field _source-fk {:table_id (:id source)+ :name "user_id"+ :database_type "INTEGER"+ :base_type :type/Integer+ :semantic_type :type/FK+ :fk_target_field_id (:id target-id)}]+ (let [mp (lib-be/application-database-metadata-provider (:id db))+ metric-query (-> (lib/query mp (lib.metadata/table mp (:id source)))+ (lib/aggregate (lib/count)))]+ (mt/with-temp [:model/Card {metric-id :id} {:dataset_query metric-query+ :database_id (:id db)+ :table_id (:id source)+ :name "Readable source metric"+ :type :metric}]+ (mt/with-no-data-perms-for-all-users!+ (perms/set-database-permission! (perms-group/all-users) db :perms/view-data :unrestricted)+ (perms/set-table-permission! (perms-group/all-users) source+ :perms/create-queries :query-builder-and-native)+ (mt/with-current-user (mt/user->id :rasta)+ (let [output (:structured-output+ (entity-details/get-metric-details {:metric-id metric-id+ :with-field-values? false}))+ dimensions (:queryable-dimensions output)+ source-fk (some #(when (= "user_id" (:name %)) %) dimensions)]+ (is (= "orders" (:base_table_name output)))+ (is (some? source-fk) "the readable source FK column is returned")+ (is (not (contains? source-fk :fk_target_portable_fk)))+ (is (not (str/includes? (pr-str output) "secret_table")))+ (is (not (str/includes? (pr-str output) "secret_id")))))))))))+;;; ============================================================;;; Portable entity_id in card details (step 11.2);;; ============================================================diff --git a/test/metabase/metabot/tools/search_test.clj b/test/metabase/metabot/tools/search_test.cljindex 55d301bf487..d8268543c5c 100644--- a/test/metabase/metabot/tools/search_test.clj+++ b/test/metabase/metabot/tools/search_test.clj@@ -621,6 +621,28 @@(is (= [db-name (:schema orders-t) (:name orders-t)](:base_table_portable_fk metric-res))))))))))+(deftest enrich-with-metric-base-tables-respects-table-permissions-test+ (testing "a readable metric does not reveal metadata for an unreadable base table"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-test-user :rasta+ (search.tu/with-temp-index-table+ (mt/with-temp [:model/Card {metric-id :id} {:name "Restricted Base Table Metric"+ :type :metric+ :database_id (mt/id)+ :table_id (mt/id :orders)+ :dataset_query {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :orders)+ :aggregation [[:count]]}}}]+ (let [results (search/search {:term-queries ["Restricted Base Table Metric"]})+ metric-res (some #(when (= [metric-id "metric"] [(:id %) (:type %)]) %) results)]+ (is (some? metric-res) "collection access still makes the metric searchable")+ (is (not-any? #(contains? metric-res %)+ [:base_table_id+ :base_table_name+ :base_table_schema+ :base_table_portable_fk])))))))))+(deftest remove-unreadable-transforms-test(testing "remove-unreadable-transforms correctly filters transforms based on source database access"(mt/with-premium-features #{:transforms-basic}