The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Optimize graph API endpoints (#65172)
metamben7937c6888cbc
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 9783490990f..81ebd29e725 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1735,6 +1735,7 @@lib-bemodelsnative-query-snippets+ permissionspremium-featuresqueriesquery-processordiff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.cljindex 138ca91fe48..497f4fd51fb 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj@@ -8,12 +8,14 @@[metabase.api.macros :as api.macros][metabase.api.routes.common :refer [+auth]][metabase.api.util.handlers :as handlers]+ [metabase.collections.models.collection :as collection][metabase.collections.models.collection.root :as collection.root][metabase.graph.core :as graph][metabase.lib-be.core :as lib-be][metabase.lib.metadata :as lib.metadata][metabase.models.interface :as mi][metabase.native-query-snippets.core :as native-query-snippets]+ [metabase.permissions.core :as perms][metabase.queries.schema :as queries.schema][metabase.revisions.core :as revisions][metabase.util.i18n :refer [tru]]@@ -166,37 +168,100 @@:dependents_count (usages [entity-type id])})(def ^:private entity-model- {:table :model/Table- :card :model/Card- :snippet :model/NativeQuerySnippet+ {:table :model/Table+ :card :model/Card+ :snippet :model/NativeQuerySnippet:transform :model/Transform:dashboard :model/Dashboard- :document :model/Document- :sandbox :model/Sandbox})+ :document :model/Document+ :sandbox :model/Sandbox})-(defn- readable-node?- "Check if the user can read the entity represented by `[entity-type entity-id]`."- [[entity-type entity-id]]- (if-let [model (entity-model entity-type)]- (try- (mi/can-read? model entity-id)- (catch Exception e- (log/debugf e "Couldn't determine if %s with id %s is readble"- (some-> entity-type name) entity-id)- false))- (do- (log/warn "Cannot determine the model of" (some-> entity-type name))- false)))+;; IMPORTANT: This map defines which fields to select when fetching entities for the dependency graph.+;; These field lists MUST be kept in sync with the frontend type definitions in:+;; frontend/src/metabase-types/api/dependencies.ts+;; (See CardDependencyNodeData, DashboardDependencyNodeData, etc.)+;;+;; Note: Some fields (like :creator, :collection, :moderation_reviews) are added via t2/hydrate,+;; and others (like :last-edit-info, :view_count) are computed/added separately.+;; This map only lists the base database columns to SELECT.+(def ^:private entity-select-fields+ {:card [:id :name :description :type :display :database_id :collection_id :dashboard_id :result_metadata+ :created_at :creator_id+ ;; :card_schema always has to be selected+ :card_schema]+ :dashboard [:id :name :description :created_at :creator_id :collection_id]+ :document [:id :name :created_at :creator_id :collection_id]+ :table [:id :name :description :display_name :db_id :schema]+ :transform [:id :name :description]+ :snippet [:id :name :description]+ :sandbox [:id :table_id]})++(defn- visible-entities-filter-clause+ "Returns a HoneySQL WHERE clause for filtering dependency graph entities by user visibility.++ Accepts two arguments:+ - `entity-type-field`: Database column name for entity type (e.g., :to_entity_type)+ - `entity-id-field`: Database column name for entity ID (e.g., :to_entity_id)++ Returns a compound [:or ...] clause checking whether entities at those columns are readable.++ Handles different entity types:+ - Superuser-only (:model/Transform, :model/Sandbox): Only if api/*is-superuser?* is true+ - Collection-based (:model/Card, :model/Dashboard, :model/Document, :model/NativeQuerySnippet):+ Uses collection/visible-collection-filter-clause. Native query snippets have additional+ restrictions for sandboxed users.+ - Table: Uses mi/visible-filter-clause with appropriate permissions"+ [entity-type-field entity-id-field]+ (into [:or]+ (keep (fn [[entity-type model]]+ (let [table-name (t2/table-name model)+ id-column (keyword (name table-name) "id")]+ (case model+ ;; Superuser-only entities+ (:model/Transform :model/Sandbox)+ (when api/*is-superuser?*+ [:and+ [:= entity-type-field (name entity-type)]+ [:in entity-id-field {:select [:id] :from [table-name]}]])++ ;; Collection-based entities+ (:model/Card :model/Dashboard :model/Document :model/NativeQuerySnippet)+ (when-not (and (= model :model/NativeQuerySnippet)+ (or (perms/sandboxed-user?)+ (not (perms/user-has-any-perms-of-type?+ api/*current-user-id* :perms/create-queries))))+ [:and+ [:= entity-type-field (name entity-type)]+ [:in entity-id-field {:select [:id]+ :from [table-name]+ :where (collection/visible-collection-filter-clause+ (keyword (name table-name) "collection_id")+ {}+ {:current-user-id api/*current-user-id*+ :is-superuser? api/*is-superuser?*})}]])++ ;; Table with visible-filter-clause+ :model/Table+ [:and+ [:= entity-type-field (name entity-type)]+ [:in entity-id-field {:select [:id]+ :from [table-name]+ :where (mi/visible-filter-clause+ model+ id-column+ {:user-id api/*current-user-id*+ :is-superuser? api/*is-superuser?*}+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder})}]]))))+ entity-model))(defn- readable-graph-dependencies- "Return a permission-aware dependency graph that only includes readable entities."[]- (dependency/filtered-graph-dependencies readable-node?))+ (dependency/filtered-graph-dependencies visible-entities-filter-clause))(defn- readable-graph-dependents- "Return a permission-aware dependent graph that only includes readable entities."[]- (dependency/filtered-graph-dependents readable-node?))+ (dependency/filtered-graph-dependents visible-entities-filter-clause))(defn- calc-usages"Calculates the count of direct dependents for all nodes in `nodes`, based on `graph`. "@@ -224,20 +289,21 @@nodes-by-type (->> (group-by first nodes)(m/map-vals #(map second %)))](mapcat (fn [[entity-type entity-ids]]- (->> (cond-> (t2/select (entity-model entity-type)- :id [:in entity-ids])- (= entity-type :card) (-> (t2/hydrate :creator :dashboard [:collection :is_personal] :moderation_reviews)- (->> (map collection.root/hydrate-root-collection))- (revisions/with-last-edit-info :card))- (= entity-type :table) (t2/hydrate :fields :db)- (= entity-type :transform) (t2/hydrate :table-with-db-and-fields)- (= entity-type :dashboard) (-> (t2/hydrate :creator [:collection :is_personal] :moderation_reviews)- (->> (map collection.root/hydrate-root-collection))- (revisions/with-last-edit-info :dashboard))- (= entity-type :document) (-> (t2/hydrate :creator [:collection :is_personal])- (->> (map collection.root/hydrate-root-collection)))- (= entity-type :sandbox) (t2/hydrate [:table :db :fields]))- (mapv #(entity-value entity-type % usages))))+ (let [model (entity-model entity-type)+ fields (entity-select-fields entity-type)]+ (->> (cond-> (t2/select (into [model] fields) :id [:in entity-ids])+ (= entity-type :card) (-> (t2/hydrate :creator :dashboard [:collection :is_personal] :moderation_reviews)+ (->> (map collection.root/hydrate-root-collection))+ (revisions/with-last-edit-info :card))+ (= entity-type :table) (t2/hydrate :fields :db)+ (= entity-type :transform) (t2/hydrate :table-with-db-and-fields)+ (= entity-type :dashboard) (-> (t2/hydrate :creator [:collection :is_personal] :moderation_reviews)+ (->> (map collection.root/hydrate-root-collection))+ (revisions/with-last-edit-info :dashboard))+ (= entity-type :document) (-> (t2/hydrate :creator [:collection :is_personal])+ (->> (map collection.root/hydrate-root-collection)))+ (= entity-type :sandbox) (t2/hydrate [:table :db :fields]))+ (mapv #(entity-value entity-type % usages)))))nodes-by-type)))(api.macros/defendpoint :get "/graph"@@ -249,16 +315,17 @@[:id {:optional true} ms/PositiveInt][:type {:optional true} (ms/enum-decode-keyword (vec (keys entity-model)))]]](api/read-check (entity-model type) id)- (let [starting-nodes [[type id]]- upstream-graph (readable-graph-dependencies)- ;; cache the downstream graph specifically, because between calculating transitive children and calculating- ;; edges, we'll call this multiple times on the same nodes.- downstream-graph (graph/cached-graph (readable-graph-dependents))- nodes (into (set starting-nodes)- (graph/transitive upstream-graph starting-nodes))- edges (graph/calc-edges-between downstream-graph nodes)]- {:nodes (expanded-nodes downstream-graph nodes)- :edges edges}))+ (lib-be/with-metadata-provider-cache+ (let [starting-nodes [[type id]]+ upstream-graph (readable-graph-dependencies)+ ;; cache the downstream graph specifically, because between calculating transitive children and calculating+ ;; edges, we'll call this multiple times on the same nodes.+ downstream-graph (graph/cached-graph (readable-graph-dependents))+ nodes (into (set starting-nodes)+ (graph/transitive upstream-graph starting-nodes))+ edges (graph/calc-edges-between downstream-graph nodes)]+ {:nodes (expanded-nodes downstream-graph nodes)+ :edges edges})))(def ^:private dependents-args[:map@@ -275,13 +342,14 @@[_route-params{:keys [id type dependent_type dependent_card_type]} :- dependents-args](api/read-check (entity-model type) id)- (let [downstream-graph (graph/cached-graph (readable-graph-dependents))- nodes (-> (graph/children-of downstream-graph [[type id]])- (get [type id]))]- (->> (expanded-nodes downstream-graph nodes)- (filter #(and (= (:type %) dependent_type)- (or (not= dependent_type :card)- (= (-> % :data :type) dependent_card_type)))))))+ (lib-be/with-metadata-provider-cache+ (let [downstream-graph (graph/cached-graph (readable-graph-dependents))+ nodes (-> (graph/children-of downstream-graph [[type id]])+ (get [type id]))]+ (->> (expanded-nodes downstream-graph nodes)+ (filter #(and (= (:type %) dependent_type)+ (or (not= dependent_type :card)+ (= (-> % :data :type) dependent_card_type))))))))(def ^{:arglists '([request respond raise])} routes"`/api/ee/dependencies` routes."diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.cljindex 76bc5b09d92..1b1341e42f8 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj@@ -21,30 +21,55 @@{:from_entity_type mi/transform-keyword:to_entity_type mi/transform-keyword})-(defn- deps-children [src-type src-id dst-type dst-id key-seq]- ;; Group all keys with the same type together, so we make O(types) indexed [[t2/select]] calls, not O(n).- (transduce (map (fn [[entity-type entity-keys]]- (let [deps (t2/select :model/Dependency- src-type entity-type- src-id [:in entity-keys])]- (u/group-by (juxt src-type src-id)- (juxt dst-type dst-id)- conj #{}- deps))))- merge {}- (u/group-by first second key-seq)))+(defn- deps-children+ "Get dependency children with optional database-level filtering.++ Returns a map from [src-type src-id] tuples to sets of [dst-type dst-id] tuples representing dependencies.++ When `destination-filter-fn` is provided, it should be a function accepting two arguments+ (entity-type-field, entity-id-field) and returning a HoneySQL WHERE clause for filtering destination entities."+ ([src-type src-id dst-type dst-id key-seq]+ (deps-children src-type src-id dst-type dst-id key-seq nil))+ ([src-type src-id dst-type dst-id key-seq destination-filter-fn]+ (let [base-filter (cond-> [:and]+ destination-filter-fn (conj (destination-filter-fn dst-type dst-id)))]+ (transduce (map (fn [[entity-type entity-keys]]+ (let [full-filter (conj base-filter+ [:= src-type (name entity-type)]+ [:in src-id entity-keys])+ deps (t2/select :model/Dependency {:where full-filter})]+ (u/group-by (juxt src-type src-id)+ (juxt dst-type dst-id)+ conj #{}+ deps))))+ merge {}+ (u/group-by first second key-seq)))))(defn- key-dependents- "Get the dependent entity keys for the entity keys in `entity-keys`.- Entity keys are [entity-type, entity-id] pairs. See [[entity-type->model]]."- [key-seq]- (deps-children :to_entity_type :to_entity_id :from_entity_type :from_entity_id key-seq))+ "Get the dependent entity keys for the entity keys in `key-seq`.++ Entity keys are [entity-type entity-id] tuples. Returns a map from source entity keys+ to sets of dependent (downstream) entity keys.++ When `destination-filter-fn` is provided, it should be a function accepting two arguments+ (entity-type-field, entity-id-field) and returning a HoneySQL WHERE clause for filtering."+ ([key-seq]+ (key-dependents key-seq nil))+ ([key-seq destination-filter-fn]+ (deps-children :to_entity_type :to_entity_id :from_entity_type :from_entity_id key-seq destination-filter-fn)))(defn- key-dependencies- "Get the dependency entity keys for the entity keys in `entity-keys`, a seq of keys.- Entity keys are [entity-type, entity-id] pairs. See [[entity-type->model]]."- [key-seq]- (deps-children :from_entity_type :from_entity_id :to_entity_type :to_entity_id key-seq))+ "Get the dependency entity keys for the entity keys in `key-seq`.++ Entity keys are [entity-type entity-id] tuples. Returns a map from source entity keys+ to sets of dependency (upstream) entity keys.++ When `destination-filter-fn` is provided, it should be a function accepting two arguments+ (entity-type-field, entity-id-field) and returning a HoneySQL WHERE clause for filtering."+ ([key-seq]+ (key-dependencies key-seq nil))+ ([key-seq destination-filter-fn]+ (deps-children :from_entity_type :from_entity_id :to_entity_type :to_entity_id key-seq destination-filter-fn)))(p/deftype+ DependencyGraph [children-fn]graph/Graph@@ -61,27 +86,35 @@[](->DependencyGraph key-dependencies))-(defn filtered-graph-dependencies- "Return a dependency graph for finding dependencies (upstream entities), filtered by `node-predicate`.- The predicate receives entity keys as `[entity-type entity-id]` and should return true for nodes to include."- [node-predicate]+(defn- filtered-graph+ "Create a dependency graph with database-level filtering.++ Arguments:+ - `key-fn`: Either key-dependencies or key-dependents, determining graph direction+ - `destination-filter-fn`: Function accepting (entity-type-field, entity-id-field)+ and returning a HoneySQL WHERE clause"+ [key-fn destination-filter-fn](->DependencyGraph(fn [key-seq]- (into {}- (map (fn [[k v]]- [k (filterv node-predicate v)]))- (key-dependencies key-seq)))))+ (key-fn key-seq destination-filter-fn))))++(defn filtered-graph-dependencies+ "Create a permission-aware dependency graph for finding upstream dependencies.++ Arguments:+ - `destination-filter-fn`: Function accepting (entity-type-field, entity-id-field)+ and returning a HoneySQL WHERE clause for filtering destination entities"+ [destination-filter-fn]+ (filtered-graph key-dependencies destination-filter-fn))(defn filtered-graph-dependents- "Return a dependency graph for finding dependents (downstream entities), filtered by `node-predicate`.- The predicate receives entity keys as `[entity-type entity-id]` and should return true for nodes to include."- [node-predicate]- (->DependencyGraph- (fn [key-seq]- (into {}- (map (fn [[k v]]- [k (filterv node-predicate v)]))- (key-dependents key-seq)))))+ "Create a permission-aware dependency graph for finding downstream dependents.++ Arguments:+ - `destination-filter-fn`: Function accepting (entity-type-field, entity-id-field)+ and returning a HoneySQL WHERE clause for filtering destination entities"+ [destination-filter-fn]+ (filtered-graph key-dependents destination-filter-fn))(defn transitive-dependents"Given a map of updated entities `{entity-type [{:id 1, ...} ...]}`, return a map of its transitive dependentsdiff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.cljindex 3b4224851fa..3060c2cf690 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj@@ -3,6 +3,7 @@[clojure.set :as set][clojure.test :refer [deftest is testing]][metabase-enterprise.dependencies.models.dependency :as deps.graph]+ [metabase.graph.core :as graph][metabase.lib.core :as lib][metabase.queries.models.card :as card][metabase.test :as mt]@@ -223,3 +224,71 @@(let [card (card/create-card! (basic-orders) user)](is (= deps.graph/current-dependency-analysis-version(t2/select-one-fn :dependency_analysis_version :model/Card :id (:id card))))))))))++(deftest filtered-graph-dependencies-test+ (testing "filtered-graph-dependencies respects filter clause"+ (mt/with-temp [:model/User user {:email "[email protected]"}]+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/Card :model/Dependency]+ (let [{id1 :id :as card1} (card/create-card! (basic-orders) user)+ {id2 :id :as card2} (card/create-card! (wrap-card card1) user)+ {id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]+ (testing "without filter, returns all dependencies"+ (let [graph (deps.graph/graph-dependencies)+ deps (graph/transitive graph [[:card id3]])]+ (is (= #{[:card id2] [:card id1] [:table (mt/id :orders)]}+ (set deps)))))+ (testing "with filter excluding card1, omits card1 and its dependencies"+ (let [filter-fn (fn [entity-type-field entity-id-field]+ [:and+ [:= entity-type-field "card"]+ [:in entity-id-field [id2 id3]]])+ graph (deps.graph/filtered-graph-dependencies filter-fn)+ deps (graph/transitive graph [[:card id3]])]+ (is (= #{[:card id2]}+ (set deps))+ "Should only include card2, not card1 or table")))+ (testing "with filter excluding card2, breaks the chain"+ (let [filter-fn (fn [entity-type-field entity-id-field]+ [:and+ [:= entity-type-field "card"]+ [:in entity-id-field [id1 id3]]])+ graph (deps.graph/filtered-graph-dependencies filter-fn)+ deps (graph/transitive graph [[:card id3]])]+ (is (= #{}+ (set deps))+ "Should be empty, chain is broken at card2")))))))))++(deftest filtered-graph-dependents-test+ (testing "filtered-graph-dependents respects filter clause"+ (mt/with-temp [:model/User user {:email "[email protected]"}]+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/Card :model/Dependency]+ (let [{id1 :id :as card1} (card/create-card! (basic-orders) user)+ {id2 :id :as card2} (card/create-card! (wrap-card card1) user)+ {id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]+ (testing "without filter, returns all dependents"+ (let [graph (deps.graph/graph-dependents)+ deps (graph/transitive graph [[:card id1]])]+ (is (= #{[:card id2] [:card id3]}+ (set deps)))))+ (testing "with filter excluding card3, omits card3"+ (let [filter-fn (fn [entity-type-field entity-id-field]+ [:and+ [:= entity-type-field "card"]+ [:in entity-id-field [id1 id2]]])+ graph (deps.graph/filtered-graph-dependents filter-fn)+ deps (graph/transitive graph [[:card id1]])]+ (is (= #{[:card id2]}+ (set deps))+ "Should only include card2, not card3")))+ (testing "with filter excluding card2, breaks the chain"+ (let [filter-fn (fn [entity-type-field entity-id-field]+ [:and+ [:= entity-type-field "card"]+ [:in entity-id-field [id1 id3]]])+ graph (deps.graph/filtered-graph-dependents filter-fn)+ deps (graph/transitive graph [[:card id1]])]+ (is (= #{}+ (set deps))+ "Should be empty, chain is broken at card2")))))))))diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.cljindex a5a478b7f6a..d9057c22680 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj@@ -1225,8 +1225,9 @@{:request-options {:headers {"x-metabase-session" crowberto-ai-token}}}{:conversation_id conversation-id})(update :structured_output (fn [output]- (filter #(#{(:id t1) (:id t2) (:id t3)} (:id %))- output)))))))))))+ (->> output+ (filter #(#{(:id t1) (:id t2) (:id t3)} (:id %)))+ (sort-by :id))))))))))))(deftest get-transform-test(mt/with-premium-features #{:metabot-v3 :transforms}diff --git a/frontend/src/metabase-types/api/dependencies.ts b/frontend/src/metabase-types/api/dependencies.tsindex b606c3b2c37..cce2010315b 100644--- a/frontend/src/metabase-types/api/dependencies.ts+++ b/frontend/src/metabase-types/api/dependencies.ts@@ -1,3 +1,8 @@+// IMPORTANT: The field selections in the *DependencyNodeData types below+// MUST be kept in sync with the backend field selections in:+// enterprise/backend/src/metabase_enterprise/dependencies/api.clj+// (See entity-select-fields map)+import type { Card, CardType } from "./card";import type { Dashboard } from "./dashboard";import type { Document } from "./document";