The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve indexed-entity handling in xrays (#82699)
Nathan Voxland280c1e7be206
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 0c8b898e758..6792268c3bc 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1108,6 +1108,7 @@indexed-entities{:team "UX West":api #{metabase.indexed-entities.api+ metabase.indexed-entities.coremetabase.indexed-entities.init}:uses #{analyticsapi@@ -3341,6 +3342,7 @@collectionsdashboardsdriver+ indexed-entitiesliblib.belib.legacy-mbql@@ -3365,7 +3367,6 @@:model/Database:model/Field:model/ModelIndex- :model/ModelIndexValue:model/Query:model/Revision:model/Segmentdiff --git a/src/metabase/indexed_entities/core.clj b/src/metabase/indexed_entities/core.cljnew file mode 100644index 00000000000..612374de08a--- /dev/null+++ b/src/metabase/indexed_entities/core.clj@@ -0,0 +1,11 @@+(ns metabase.indexed-entities.core+ "API namespace for the indexed-entities module."+ (:require+ [metabase.indexed-entities.models.model-index]+ [potemkin :as p]))++(comment metabase.indexed-entities.models.model-index/keep-me)++(p/import-vars+ [metabase.indexed-entities.models.model-index+ value-for-pk])diff --git a/src/metabase/indexed_entities/models/model_index.clj b/src/metabase/indexed_entities/models/model_index.cljindex d06bdebd199..10698d86fbc 100644--- a/src/metabase/indexed_entities/models/model_index.clj+++ b/src/metabase/indexed_entities/models/model_index.clj@@ -86,8 +86,11 @@{:field-ref field-ref:valid-clauses [:field :expression]}))))-(mu/defn ^:private fetch-values- [model-index :- ::indexed-entities.schema/model-index]+(mu/defn- values-query+ "The query selecting `[pk value]` tuples from `model-index`'s model: every record when `pk` is nil, otherwise just+ the record with that primary key."+ [model-index :- ::indexed-entities.schema/model-index+ pk :- [:maybe :int]](let [model (indexed-entities.db/card (:model_id model-index))fix (mu/fn [field-ref :- ::mbql.s/FieldOrExpressionRefbase-type :- ::lib.schema.common/base-type]@@ -96,18 +99,38 @@;; :type/Text and :type/Integer are ensured at creation time on the api.value-ref (-> model-index :value_ref (fix :type/Text))pk-ref (-> model-index :pk_ref (fix :type/Integer))]- (try- [nil (->> (qp/process-query- ;; TODO (Cam 10/1/25) -- update this to generate the query using Lib- {:database (:database_id model)- :type :query- :query {:source-table (format "card__%d" (:id model))- :breakout [pk-ref value-ref]- :limit (inc max-indexed-values)}})- :data :rows (filter valid-tuples?))]- (catch Exception e- (log/warnf "Error fetching indexed values for model %s: %s" (:id model) (ex-message e))- [(ex-message e) []]))))+ ;; TODO (Cam 10/1/25) -- update this to generate the query using Lib+ {:database (:database_id model)+ :type :query+ :query (cond-> {:source-table (format "card__%d" (:id model))+ :breakout [pk-ref value-ref]+ :limit (inc max-indexed-values)}+ pk (assoc :filter [:= pk-ref pk]+ :limit 1))}))++(mu/defn ^:private fetch-values+ [model-index :- ::indexed-entities.schema/model-index]+ (try+ [nil (->> (qp/process-query (values-query model-index nil))+ :data :rows (filter valid-tuples?))]+ (catch Exception e+ (log/warnf "Error fetching indexed values for model %s: %s" (:model_id model-index) (ex-message e))+ [(ex-message e) []])))++(mu/defn value-for-pk :- [:maybe :string]+ "The indexed value of the record of `model-index`'s model whose primary key is `pk`, as the current user sees it:+ the model is queried through the QP, so data permissions, sandboxing, impersonation and routing all apply. This+ deliberately never reads `model_index_value`, which is a lens-free copy shared by every user and reachable only+ through search. Nil when no record matches or the user may not run the model."+ [model-index :- ::indexed-entities.schema/model-index+ pk :- :int]+ (try+ (let [[[_pk value]] (->> (qp/process-query (values-query model-index pk))+ :data :rows (filter valid-tuples?))]+ (some-> value str))+ (catch Exception e+ (log/debugf "Could not read indexed value %s of model %s: %s" pk (:model_id model-index) (ex-message e))+ nil)))(defn find-changes"Find additions and deletions in indexed values. `source-values` are from the db, `indexed-values` are what wediff --git a/src/metabase/xrays/api/automagic_dashboards.clj b/src/metabase/xrays/api/automagic_dashboards.cljindex 0b2aba8dfb5..c07316de5a3 100644--- a/src/metabase/xrays/api/automagic_dashboards.clj+++ b/src/metabase/xrays/api/automagic_dashboards.clj@@ -5,6 +5,7 @@[medley.core :as m][metabase.api.common :as api][metabase.api.macros :as api.macros]+ [metabase.indexed-entities.core :as indexed-entities][metabase.lib-be.core :as lib-be][metabase.lib-be.schema :as lib-be.schema][metabase.lib.core :as lib]@@ -293,11 +294,18 @@:description nil}}}}cards)))+(defn- linked-dashboard-name+ [model-name model-pk indexed-value]+ (if indexed-value+ (format "Here's a look at \"%s\" from \"%s\"" indexed-value model-name)+ (format "Here's a look at \"%s\" #%s" model-name model-pk)))+(defn- create-linked-dashboard- "For each joinable table from `model`, create an x-ray dashboard as a tab."- [{{indexed-entity-name :name :keys [model_pk]} :model-index-value- {model-name :name :as model} :model- :keys [linked-tables]}]+ "For each joinable table from `model`, create an x-ray dashboard as a tab, filtered on the record whose primary key+ is `model-pk`. `indexed-value` is that record's display value as the requesting user sees it, or nil when they may+ not read it; it only ever decorates the name and description."+ [{{model-name :name :as model} :model+ :keys [linked-tables model-pk indexed-value]}](if (seq linked-tables)(let [child-dashboards (map (fn [{:keys [linked-table-id linked-field-id]}](let [table (xrays.db/table linked-table-id)@@ -305,12 +313,13 @@(automagic-dashboards.core/automagic-analysistable{:show :all- :query-filter [(lib/= (lib.metadata/field mp linked-field-id) model_pk)]})))+ :query-filter [(lib/= (lib.metadata/field mp linked-field-id) model-pk)]})))linked-tables)seed-dashboard (-> (first child-dashboards)(merge- {:name (format "Here's a look at \"%s\" from \"%s\"" indexed-entity-name model-name)- :description (format "A dashboard focusing on information linked to %s" indexed-entity-name)+ {:name (linked-dashboard-name model-name model-pk indexed-value)+ :description (format "A dashboard focusing on information linked to %s"+ (or indexed-value (format "\"%s\" #%s" model-name model-pk))):parameters []:param_fields {}})(dissoc :transient_name@@ -337,7 +346,7 @@:tabs []})))(update seed-dashboard:dashcards (fn [cards] (add-source-model-link model cards)))))- {:name (format "Here's a look at \"%s\" from \"%s\"" indexed-entity-name model-name)+ {:name (linked-dashboard-name model-name model-pk indexed-value):dashcards (add-source-model-linkmodel[{:row 0@@ -357,23 +366,24 @@#_{:clj-kondo/ignore [:metabase/validate-defendpoint-route-uses-kebab-case:metabase/validate-defendpoint-has-response-schema]}(api.macros/defendpoint :get "/model_index/:model-index-id/primary_key/:pk-id"- "Return an automagic dashboard for an entity detail specified by `entity`- with id `id` and a primary key of `indexed-value`."+ "Return an automagic dashboard for the record of the model indexed by `model-index-id` whose primary key is+ `pk-id`. The record's value, used in the title, is read through the QP as the requesting user (never from+ `model_index_value`, which is not permission-checked); a `pk-id` they cannot resolve yields a dashboard titled+ by the pk, with no matching rows, rather than a 404."[{:keys [model-index-id pk-id]} :- [:map {:closed true}[:model-index-id :int][:pk-id :int]]](api/let-404 [model-index (xrays.db/model-index model-index-id)- model (xrays.db/card (:model_id model-index))- model-index-value (xrays.db/model-index-value model-index-id pk-id)]+ model (xrays.db/card (:model_id model-index))];; `->entity` does a read check on the model but this is here as well to be extra sure.(api/read-check :model/Card (:model_id model-index))- (let [linked (linked-entities {:model model- :model-index model-index- :model-index-value model-index-value})]- (create-linked-dashboard {:model model- :linked-tables linked- :model-index model-index- :model-index-value model-index-value}))))+ (let [linked (linked-entities {:model model+ :model-index model-index})]+ (create-linked-dashboard {:model model+ :linked-tables linked+ :model-index model-index+ :model-pk pk-id+ :indexed-value (indexed-entities/value-for-pk model-index pk-id)}))));; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to;; use our API + we will need it when we make auto-TypeScript-signature generation happendiff --git a/src/metabase/xrays/db.clj b/src/metabase/xrays/db.cljindex dd7295d14ab..7ca070af3d7 100644--- a/src/metabase/xrays/db.clj+++ b/src/metabase/xrays/db.clj@@ -274,9 +274,3 @@"The ModelIndex with `model-index-id`, or nil."[model-index-id :- ms/PositiveInt](t2/select-one :model/ModelIndex model-index-id))--(mu/defn model-index-value- "The ModelIndexValue of the ModelIndex with `model-index-id` for the model primary key `model-pk`, or nil."- [model-index-id :- ms/PositiveInt- model-pk :- :int]- (t2/select-one :model/ModelIndexValue :model_index_id model-index-id :model_pk model-pk))diff --git a/test/metabase/indexed_entities/models/model_index_test.clj b/test/metabase/indexed_entities/models/model_index_test.cljindex c49b59c4083..ae3cec7f444 100644--- a/test/metabase/indexed_entities/models/model_index_test.clj+++ b/test/metabase/indexed_entities/models/model_index_test.clj@@ -1,5 +1,6 @@(ns ^:mb/driver-tests metabase.indexed-entities.models.model-index-test- {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.indexed-entities.models.model-index-test]}}}}}}+ {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.indexed-entities.models.model-index-test]}+ metabase.test.data/run-mbql-query {:namespaces [metabase.indexed-entities.models.model-index-test]}}}}}}(:require[clojure.set :as set][clojure.test :refer :all]@@ -221,6 +222,28 @@(-> (mr/explain [:sequential [:tuple number? string?]] values)(me/humanize))))))))))+(deftest value-for-pk-test+ (mt/dataset test-data+ (mt/with-temp [:model/Card model (assoc (mt/card-with-source-metadata-for-query (mt/mbql-query products))+ :type :model)]+ (let [model-index {:id 1+ :model_id (u/the-id model)+ :pk_ref (mt/$ids :products $id)+ :value_ref (mt/$ids :products $title)}+ title-of (fn [pk] (-> (mt/run-mbql-query products {:filter [:= $id pk] :fields [$title]})+ mt/rows ffirst))]+ (testing "the value of the record with that pk, read through the QP as the current user"+ (mt/with-test-user :crowberto+ (is (= (title-of 1) (model-index/value-for-pk model-index 1)))+ (is (= (title-of 2) (model-index/value-for-pk model-index 2)))))+ (testing "nil when no record has that pk"+ (mt/with-test-user :crowberto+ (is (nil? (model-index/value-for-pk model-index 999999)))))+ (testing "nil, not an error, when the current user may not run the model"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-test-user :rasta+ (is (nil? (model-index/value-for-pk model-index 1))))))))))+(defn- test-index!"Takes a query, pk and value names so it can look up the exact field ref from the metadata. This is what the UI woulddo and ensures that items in the options map are correct."diff --git a/test/metabase/xrays/api/automagic_dashboards_test.clj b/test/metabase/xrays/api/automagic_dashboards_test.cljindex 57fcc20efca..bf4fd1c7b52 100644--- a/test/metabase/xrays/api/automagic_dashboards_test.clj+++ b/test/metabase/xrays/api/automagic_dashboards_test.clj@@ -1,5 +1,6 @@(ns metabase.xrays.api.automagic-dashboards-test- {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.xrays.api.automagic-dashboards-test]}}}}}}+ {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.xrays.api.automagic-dashboards-test]}+ metabase.test.data/run-mbql-query {:namespaces [metabase.xrays.api.automagic-dashboards-test]}}}}}}(:require[clojure.string :as str][clojure.test :refer :all]@@ -332,11 +333,9 @@:value-ref value-ref:creator-id (mt/user->id :crowberto)})](model-index/add-values! model-index)- (f {:model model- :model-index (t2/select-one :model/ModelIndex :id (:id model-index))- :model-index-value (t2/select-one :model/ModelIndexValue- :model_index_id (:id model-index)- :model_pk 1)})))))+ (f {:model model+ :model-index (t2/select-one :model/ModelIndex :id (:id model-index))+ :model-pk 1})))))(defmacro with-indexed-model!"Creates a model based on `query-info`, which is indexed.@@ -354,10 +353,10 @@[:fn neg-int?])(defn- expected-filters- [{:keys [model-index-value] :as info}]+ [{:keys [model-pk] :as info}](let [linked-tables (api.magic/linked-entities info)](into #{} (map (fn [{fk-id :linked-field-id}]- [:= [:field fk-id nil] (:model_pk model-index-value)]))+ [:= [:field fk-id nil] model-pk]))linked-tables)))(defn- cards-have-filters?@@ -382,22 +381,22 @@:virtual_card {:display :text},:dashcard.background false,:text.align_vertical :bottom}}]}- (#'api.magic/create-linked-dashboard {:model nil- :linked-tables ()- :model-index nil- :model-index-value nil})))))+ (#'api.magic/create-linked-dashboard {:model nil+ :linked-tables ()+ :model-index nil+ :model-pk 1})))))(deftest create-linked-dashboard-test-regular-queries(mt/dataset test-data(testing "x-ray an mbql model"- (with-indexed-model! [{:keys [model model-index model-index-value]}+ (with-indexed-model! [{:keys [model model-index model-pk]}{:query (mt/mbql-query products):pk-ref (mt/$ids :products $id):value-ref (mt/$ids :products $title)}](let [dash (#'api.magic/create-linked-dashboard- {:model model- :model-index model-index- :model-index-value model-index-value+ {:model model+ :model-index model-index+ :model-pk model-pk:linked-tables (mt/$ids [{:linked-table-id $$reviews:linked-field-id %reviews.product_id}{:linked-table-id $$orders@@ -420,16 +419,14 @@(group-by :dashboard_tab_id)vals(map first)))))- (testing "The generated dashboard has a meaningful name and description"- (is (true?- (and- (str/includes? (:name dash) (:name model))- (str/includes? (:name dash) (:name model-index-value)))))- (is (true? (str/includes? (:description dash) (:name model-index-value)))))+ (testing "Without an indexed value the dashboard is named after the model and the pk"+ (is (str/includes? (:name dash) (:name model)))+ (is (str/includes? (:name dash) (str model-pk)))+ (is (str/includes? (:description dash) (:name model))))(testing "All query cards have the correct filters"- (let [pk-filters (expected-filters {:model model- :model-index model-index- :model-index-value model-index-value})]+ (let [pk-filters (expected-filters {:model model+ :model-index model-index+ :model-pk model-pk})](cards-have-filters? (:dashcards dash) pk-filters))))))(testing "X-ray a native model"(letfn [(lower [x] (u/lower-case-en x))@@ -450,7 +447,7 @@id-field-ref (:field_ref (by-id results-meta "id"))title-field-ref (:field_ref (by-id results-meta "title"))id-field-id (mt/id :products :id)]- (with-indexed-model! [{:keys [model model-index model-index-value]}+ (with-indexed-model! [{:keys [model model-index model-pk]}{:query (mt/native-query {:query "select * from products"}):pk-ref id-field-ref:value-ref title-field-ref}]@@ -465,9 +462,9 @@"Metadata not updated with the mapping to the database column")(let [model (t2/select-one 'Card :id (:id model))dash (#'api.magic/create-linked-dashboard- {:model model- :model-index model-index- :model-index-value model-index-value+ {:model model+ :model-index model-index+ :model-pk model-pk:linked-tables (mt/$ids [{:linked-table-id $$reviews:linked-field-id %reviews.product_id}{:linked-table-id $$orders@@ -478,22 +475,22 @@:name "A look at Orders" :position 1}](:tabs dash)))(testing "All query cards have the correct filters"- (let [pk-filters (expected-filters {:model model- :model-index model-index- :model-index-value model-index-value})]+ (let [pk-filters (expected-filters {:model model+ :model-index model-index+ :model-pk model-pk})](cards-have-filters? (:dashcards dash) pk-filters))))))))))(deftest create-linked-dashboard-test-single-link(mt/dataset test-data(testing "with only single linked table"- (with-indexed-model! [{:keys [model model-index model-index-value]}+ (with-indexed-model! [{:keys [model model-index model-pk]}{:query (mt/mbql-query people):pk-ref (mt/$ids :people $id):value-ref (mt/$ids :people $email)}](let [dash (#'api.magic/create-linked-dashboard- {:model model- :model-index model-index- :model-index-value model-index-value+ {:model model+ :model-index model-index+ :model-pk model-pk:linked-tables (mt/$ids [{:linked-table-id $$orders:linked-field-id %orders.user_id}])})];; FE has a bug where it doesn't fire off queries for cards if there's only a single tab. So we hack around@@ -508,18 +505,65 @@:model "dataset":display "table"}}}}(->> dash :dashcards first))))- (testing "The generated dashboard has a meaningful name and description"- (is (true?- (and- (str/includes? (:name dash) (:name model))- (str/includes? (:name dash) (:name model-index-value)))))- (is (true? (str/includes? (:description dash) (:name model-index-value)))))+ (testing "Without an indexed value the dashboard is named after the model and the pk"+ (is (str/includes? (:name dash) (:name model)))+ (is (str/includes? (:name dash) (str model-pk)))+ (is (str/includes? (:description dash) (:name model))))(testing "All query cards have the correct filters"- (let [pk-filters (expected-filters {:model model- :model-index model-index- :model-index-value model-index-value})]+ (let [pk-filters (expected-filters {:model model+ :model-index model-index+ :model-pk model-pk})](cards-have-filters? (:dashcards dash) pk-filters))))))))+(defn- linked-pk-filter-values+ "For every query card in `dashcards` (pMBQL over the wire), the value its `=` filter on one of `fk-field-ids`+ compares against, or nil when it has no such filter."+ [dashcards fk-field-ids]+ (for [{:keys [card]} dashcards+ :let [query (:dataset_query card)]+ :when query]+ (some (fn [[op _opts [_ _ field-id] value]]+ (when (and (= "=" op) (contains? fk-field-ids field-id))+ value))+ (-> query :stages first :filters))))++(deftest model-index-primary-key-endpoint-does-not-expose-indexed-values-test+ (testing "GET /api/automagic-dashboards/model_index/:model-index-id/primary_key/:pk-id"+ (mt/dataset test-data+ (mt/with-temp [:model/Card model {:type :model+ :dataset_query (mt/mbql-query products)}]+ (mt/with-model-cleanup [:model/ModelIndex]+ (let [model-index (model-index/create {:model-id (:id model)+ :pk-ref (mt/$ids :products $id)+ :value-ref (mt/$ids :products $title)+ :creator-id (mt/user->id :crowberto)})+ ;; a value no warehouse row carries, so its presence anywhere in a response is unambiguous+ sentinel "SENTINEL-INDEXED-VALUE-7f3a"+ url (format "automagic-dashboards/model_index/%d/primary_key/1" (:id model-index))+ fk-field-ids (mt/$ids #{%orders.product_id %reviews.product_id})]+ (model-index/add-values! model-index)+ (t2/update! :model/ModelIndexValue {:model_index_id (:id model-index) :model_pk 1} {:name sentinel})+ (testing "an admin gets a dashboard filtered on the pk from the URL, titled with the record's value as+ read through the QP -- never the cached one"+ (let [dash (mt/user-http-request :crowberto :get 200 url)+ pk-filters (linked-pk-filter-values (:dashcards dash) fk-field-ids)+ title (-> (mt/run-mbql-query products {:filter [:= $id 1] :fields [$title]}) mt/rows ffirst)]+ (is (not (str/includes? (json/encode dash) sentinel)))+ (is (= (format "Here's a look at \"%s\" from \"%s\"" title (:name model)) (:name dash)))+ (is (str/includes? (:description dash) title))+ (is (seq pk-filters))+ (is (every? #{1} pk-filters))))+ (testing "a user with read access to the model but no data permissions gets neither value: the title+ falls back to the pk"+ (mt/with-no-data-perms-for-all-users!+ (let [dash (mt/user-http-request :rasta :get 200 url)]+ (is (not (str/includes? (json/encode dash) sentinel)))+ (is (= (format "Here's a look at \"%s\" #1" (:name model)) (:name dash))))))+ (testing "a pk with no indexed value still yields a dashboard, so existence is not confirmed either"+ (is (map? (mt/user-http-request :crowberto :get 200+ (format "automagic-dashboards/model_index/%d/primary_key/999999"+ (:id model-index))))))))))))+;; ------------------------------------------------ `show` limit test -------------------------------------------------;; Historically, the used params are `nil` and "all", so this tests the integer case.