Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Optimize graph API endpoints (#65172)

metamben7937c6888cbc
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 9783490990f..81ebd29e725 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1735,6 +1735,7 @@
lib-be
models
native-query-snippets
+ permissions
premium-features
queries
query-processor
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
index 138ca91fe48..497f4fd51fb 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
@@ -8,12 +8,14 @@
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]
[metabase.api.util.handlers :as handlers]
+ [metabase.collections.models.collection :as collection]
[metabase.collections.models.collection.root :as collection.root]
[metabase.graph.core :as graph]
[metabase.lib-be.core :as lib-be]
[metabase.lib.metadata :as lib.metadata]
[metabase.models.interface :as mi]
[metabase.native-query-snippets.core :as native-query-snippets]
+ [metabase.permissions.core :as perms]
[metabase.queries.schema :as queries.schema]
[metabase.revisions.core :as revisions]
[metabase.util.i18n :refer [tru]]
@@ -166,37 +168,100 @@
:dependents_count (usages [entity-type id])})
(def ^:private entity-model
- {:table :model/Table
- :card :model/Card
- :snippet :model/NativeQuerySnippet
+ {:table :model/Table
+ :card :model/Card
+ :snippet :model/NativeQuerySnippet
:transform :model/Transform
:dashboard :model/Dashboard
- :document :model/Document
- :sandbox :model/Sandbox})
+ :document :model/Document
+ :sandbox :model/Sandbox})
-(defn- readable-node?
- "Check if the user can read the entity represented by `[entity-type entity-id]`."
- [[entity-type entity-id]]
- (if-let [model (entity-model entity-type)]
- (try
- (mi/can-read? model entity-id)
- (catch Exception e
- (log/debugf e "Couldn't determine if %s with id %s is readble"
- (some-> entity-type name) entity-id)
- false))
- (do
- (log/warn "Cannot determine the model of" (some-> entity-type name))
- false)))
+;; IMPORTANT: This map defines which fields to select when fetching entities for the dependency graph.
+;; These field lists MUST be kept in sync with the frontend type definitions in:
+;; frontend/src/metabase-types/api/dependencies.ts
+;; (See CardDependencyNodeData, DashboardDependencyNodeData, etc.)
+;;
+;; Note: Some fields (like :creator, :collection, :moderation_reviews) are added via t2/hydrate,
+;; and others (like :last-edit-info, :view_count) are computed/added separately.
+;; This map only lists the base database columns to SELECT.
+(def ^:private entity-select-fields
+ {:card [:id :name :description :type :display :database_id :collection_id :dashboard_id :result_metadata
+ :created_at :creator_id
+ ;; :card_schema always has to be selected
+ :card_schema]
+ :dashboard [:id :name :description :created_at :creator_id :collection_id]
+ :document [:id :name :created_at :creator_id :collection_id]
+ :table [:id :name :description :display_name :db_id :schema]
+ :transform [:id :name :description]
+ :snippet [:id :name :description]
+ :sandbox [:id :table_id]})
+
+(defn- visible-entities-filter-clause
+ "Returns a HoneySQL WHERE clause for filtering dependency graph entities by user visibility.
+
+ Accepts two arguments:
+ - `entity-type-field`: Database column name for entity type (e.g., :to_entity_type)
+ - `entity-id-field`: Database column name for entity ID (e.g., :to_entity_id)
+
+ Returns a compound [:or ...] clause checking whether entities at those columns are readable.
+
+ Handles different entity types:
+ - Superuser-only (:model/Transform, :model/Sandbox): Only if api/*is-superuser?* is true
+ - Collection-based (:model/Card, :model/Dashboard, :model/Document, :model/NativeQuerySnippet):
+ Uses collection/visible-collection-filter-clause. Native query snippets have additional
+ restrictions for sandboxed users.
+ - Table: Uses mi/visible-filter-clause with appropriate permissions"
+ [entity-type-field entity-id-field]
+ (into [:or]
+ (keep (fn [[entity-type model]]
+ (let [table-name (t2/table-name model)
+ id-column (keyword (name table-name) "id")]
+ (case model
+ ;; Superuser-only entities
+ (:model/Transform :model/Sandbox)
+ (when api/*is-superuser?*
+ [:and
+ [:= entity-type-field (name entity-type)]
+ [:in entity-id-field {:select [:id] :from [table-name]}]])
+
+ ;; Collection-based entities
+ (:model/Card :model/Dashboard :model/Document :model/NativeQuerySnippet)
+ (when-not (and (= model :model/NativeQuerySnippet)
+ (or (perms/sandboxed-user?)
+ (not (perms/user-has-any-perms-of-type?
+ api/*current-user-id* :perms/create-queries))))
+ [:and
+ [:= entity-type-field (name entity-type)]
+ [:in entity-id-field {:select [:id]
+ :from [table-name]
+ :where (collection/visible-collection-filter-clause
+ (keyword (name table-name) "collection_id")
+ {}
+ {:current-user-id api/*current-user-id*
+ :is-superuser? api/*is-superuser?*})}]])
+
+ ;; Table with visible-filter-clause
+ :model/Table
+ [:and
+ [:= entity-type-field (name entity-type)]
+ [:in entity-id-field {:select [:id]
+ :from [table-name]
+ :where (mi/visible-filter-clause
+ model
+ id-column
+ {:user-id api/*current-user-id*
+ :is-superuser? api/*is-superuser?*}
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder})}]]))))
+ entity-model))
(defn- readable-graph-dependencies
- "Return a permission-aware dependency graph that only includes readable entities."
[]
- (dependency/filtered-graph-dependencies readable-node?))
+ (dependency/filtered-graph-dependencies visible-entities-filter-clause))
(defn- readable-graph-dependents
- "Return a permission-aware dependent graph that only includes readable entities."
[]
- (dependency/filtered-graph-dependents readable-node?))
+ (dependency/filtered-graph-dependents visible-entities-filter-clause))
(defn- calc-usages
"Calculates the count of direct dependents for all nodes in `nodes`, based on `graph`. "
@@ -224,20 +289,21 @@
nodes-by-type (->> (group-by first nodes)
(m/map-vals #(map second %)))]
(mapcat (fn [[entity-type entity-ids]]
- (->> (cond-> (t2/select (entity-model entity-type)
- :id [:in entity-ids])
- (= entity-type :card) (-> (t2/hydrate :creator :dashboard [:collection :is_personal] :moderation_reviews)
- (->> (map collection.root/hydrate-root-collection))
- (revisions/with-last-edit-info :card))
- (= entity-type :table) (t2/hydrate :fields :db)
- (= entity-type :transform) (t2/hydrate :table-with-db-and-fields)
- (= entity-type :dashboard) (-> (t2/hydrate :creator [:collection :is_personal] :moderation_reviews)
- (->> (map collection.root/hydrate-root-collection))
- (revisions/with-last-edit-info :dashboard))
- (= entity-type :document) (-> (t2/hydrate :creator [:collection :is_personal])
- (->> (map collection.root/hydrate-root-collection)))
- (= entity-type :sandbox) (t2/hydrate [:table :db :fields]))
- (mapv #(entity-value entity-type % usages))))
+ (let [model (entity-model entity-type)
+ fields (entity-select-fields entity-type)]
+ (->> (cond-> (t2/select (into [model] fields) :id [:in entity-ids])
+ (= entity-type :card) (-> (t2/hydrate :creator :dashboard [:collection :is_personal] :moderation_reviews)
+ (->> (map collection.root/hydrate-root-collection))
+ (revisions/with-last-edit-info :card))
+ (= entity-type :table) (t2/hydrate :fields :db)
+ (= entity-type :transform) (t2/hydrate :table-with-db-and-fields)
+ (= entity-type :dashboard) (-> (t2/hydrate :creator [:collection :is_personal] :moderation_reviews)
+ (->> (map collection.root/hydrate-root-collection))
+ (revisions/with-last-edit-info :dashboard))
+ (= entity-type :document) (-> (t2/hydrate :creator [:collection :is_personal])
+ (->> (map collection.root/hydrate-root-collection)))
+ (= entity-type :sandbox) (t2/hydrate [:table :db :fields]))
+ (mapv #(entity-value entity-type % usages)))))
nodes-by-type)))
(api.macros/defendpoint :get "/graph"
@@ -249,16 +315,17 @@
[:id {:optional true} ms/PositiveInt]
[:type {:optional true} (ms/enum-decode-keyword (vec (keys entity-model)))]]]
(api/read-check (entity-model type) id)
- (let [starting-nodes [[type id]]
- upstream-graph (readable-graph-dependencies)
- ;; cache the downstream graph specifically, because between calculating transitive children and calculating
- ;; edges, we'll call this multiple times on the same nodes.
- downstream-graph (graph/cached-graph (readable-graph-dependents))
- nodes (into (set starting-nodes)
- (graph/transitive upstream-graph starting-nodes))
- edges (graph/calc-edges-between downstream-graph nodes)]
- {:nodes (expanded-nodes downstream-graph nodes)
- :edges edges}))
+ (lib-be/with-metadata-provider-cache
+ (let [starting-nodes [[type id]]
+ upstream-graph (readable-graph-dependencies)
+ ;; cache the downstream graph specifically, because between calculating transitive children and calculating
+ ;; edges, we'll call this multiple times on the same nodes.
+ downstream-graph (graph/cached-graph (readable-graph-dependents))
+ nodes (into (set starting-nodes)
+ (graph/transitive upstream-graph starting-nodes))
+ edges (graph/calc-edges-between downstream-graph nodes)]
+ {:nodes (expanded-nodes downstream-graph nodes)
+ :edges edges})))
(def ^:private dependents-args
[:map
@@ -275,13 +342,14 @@
[_route-params
{:keys [id type dependent_type dependent_card_type]} :- dependents-args]
(api/read-check (entity-model type) id)
- (let [downstream-graph (graph/cached-graph (readable-graph-dependents))
- nodes (-> (graph/children-of downstream-graph [[type id]])
- (get [type id]))]
- (->> (expanded-nodes downstream-graph nodes)
- (filter #(and (= (:type %) dependent_type)
- (or (not= dependent_type :card)
- (= (-> % :data :type) dependent_card_type)))))))
+ (lib-be/with-metadata-provider-cache
+ (let [downstream-graph (graph/cached-graph (readable-graph-dependents))
+ nodes (-> (graph/children-of downstream-graph [[type id]])
+ (get [type id]))]
+ (->> (expanded-nodes downstream-graph nodes)
+ (filter #(and (= (:type %) dependent_type)
+ (or (not= dependent_type :card)
+ (= (-> % :data :type) dependent_card_type))))))))
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/dependencies` routes."
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
index 76bc5b09d92..1b1341e42f8 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
@@ -21,30 +21,55 @@
{:from_entity_type mi/transform-keyword
:to_entity_type mi/transform-keyword})
-(defn- deps-children [src-type src-id dst-type dst-id key-seq]
- ;; Group all keys with the same type together, so we make O(types) indexed [[t2/select]] calls, not O(n).
- (transduce (map (fn [[entity-type entity-keys]]
- (let [deps (t2/select :model/Dependency
- src-type entity-type
- src-id [:in entity-keys])]
- (u/group-by (juxt src-type src-id)
- (juxt dst-type dst-id)
- conj #{}
- deps))))
- merge {}
- (u/group-by first second key-seq)))
+(defn- deps-children
+ "Get dependency children with optional database-level filtering.
+
+ Returns a map from [src-type src-id] tuples to sets of [dst-type dst-id] tuples representing dependencies.
+
+ When `destination-filter-fn` is provided, it should be a function accepting two arguments
+ (entity-type-field, entity-id-field) and returning a HoneySQL WHERE clause for filtering destination entities."
+ ([src-type src-id dst-type dst-id key-seq]
+ (deps-children src-type src-id dst-type dst-id key-seq nil))
+ ([src-type src-id dst-type dst-id key-seq destination-filter-fn]
+ (let [base-filter (cond-> [:and]
+ destination-filter-fn (conj (destination-filter-fn dst-type dst-id)))]
+ (transduce (map (fn [[entity-type entity-keys]]
+ (let [full-filter (conj base-filter
+ [:= src-type (name entity-type)]
+ [:in src-id entity-keys])
+ deps (t2/select :model/Dependency {:where full-filter})]
+ (u/group-by (juxt src-type src-id)
+ (juxt dst-type dst-id)
+ conj #{}
+ deps))))
+ merge {}
+ (u/group-by first second key-seq)))))
(defn- key-dependents
- "Get the dependent entity keys for the entity keys in `entity-keys`.
- Entity keys are [entity-type, entity-id] pairs. See [[entity-type->model]]."
- [key-seq]
- (deps-children :to_entity_type :to_entity_id :from_entity_type :from_entity_id key-seq))
+ "Get the dependent entity keys for the entity keys in `key-seq`.
+
+ Entity keys are [entity-type entity-id] tuples. Returns a map from source entity keys
+ to sets of dependent (downstream) entity keys.
+
+ When `destination-filter-fn` is provided, it should be a function accepting two arguments
+ (entity-type-field, entity-id-field) and returning a HoneySQL WHERE clause for filtering."
+ ([key-seq]
+ (key-dependents key-seq nil))
+ ([key-seq destination-filter-fn]
+ (deps-children :to_entity_type :to_entity_id :from_entity_type :from_entity_id key-seq destination-filter-fn)))
(defn- key-dependencies
- "Get the dependency entity keys for the entity keys in `entity-keys`, a seq of keys.
- Entity keys are [entity-type, entity-id] pairs. See [[entity-type->model]]."
- [key-seq]
- (deps-children :from_entity_type :from_entity_id :to_entity_type :to_entity_id key-seq))
+ "Get the dependency entity keys for the entity keys in `key-seq`.
+
+ Entity keys are [entity-type entity-id] tuples. Returns a map from source entity keys
+ to sets of dependency (upstream) entity keys.
+
+ When `destination-filter-fn` is provided, it should be a function accepting two arguments
+ (entity-type-field, entity-id-field) and returning a HoneySQL WHERE clause for filtering."
+ ([key-seq]
+ (key-dependencies key-seq nil))
+ ([key-seq destination-filter-fn]
+ (deps-children :from_entity_type :from_entity_id :to_entity_type :to_entity_id key-seq destination-filter-fn)))
(p/deftype+ DependencyGraph [children-fn]
graph/Graph
@@ -61,27 +86,35 @@
[]
(->DependencyGraph key-dependencies))
-(defn filtered-graph-dependencies
- "Return a dependency graph for finding dependencies (upstream entities), filtered by `node-predicate`.
- The predicate receives entity keys as `[entity-type entity-id]` and should return true for nodes to include."
- [node-predicate]
+(defn- filtered-graph
+ "Create a dependency graph with database-level filtering.
+
+ Arguments:
+ - `key-fn`: Either key-dependencies or key-dependents, determining graph direction
+ - `destination-filter-fn`: Function accepting (entity-type-field, entity-id-field)
+ and returning a HoneySQL WHERE clause"
+ [key-fn destination-filter-fn]
(->DependencyGraph
(fn [key-seq]
- (into {}
- (map (fn [[k v]]
- [k (filterv node-predicate v)]))
- (key-dependencies key-seq)))))
+ (key-fn key-seq destination-filter-fn))))
+
+(defn filtered-graph-dependencies
+ "Create a permission-aware dependency graph for finding upstream dependencies.
+
+ Arguments:
+ - `destination-filter-fn`: Function accepting (entity-type-field, entity-id-field)
+ and returning a HoneySQL WHERE clause for filtering destination entities"
+ [destination-filter-fn]
+ (filtered-graph key-dependencies destination-filter-fn))
(defn filtered-graph-dependents
- "Return a dependency graph for finding dependents (downstream entities), filtered by `node-predicate`.
- The predicate receives entity keys as `[entity-type entity-id]` and should return true for nodes to include."
- [node-predicate]
- (->DependencyGraph
- (fn [key-seq]
- (into {}
- (map (fn [[k v]]
- [k (filterv node-predicate v)]))
- (key-dependents key-seq)))))
+ "Create a permission-aware dependency graph for finding downstream dependents.
+
+ Arguments:
+ - `destination-filter-fn`: Function accepting (entity-type-field, entity-id-field)
+ and returning a HoneySQL WHERE clause for filtering destination entities"
+ [destination-filter-fn]
+ (filtered-graph key-dependents destination-filter-fn))
(defn transitive-dependents
"Given a map of updated entities `{entity-type [{:id 1, ...} ...]}`, return a map of its transitive dependents
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
index 3b4224851fa..3060c2cf690 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
@@ -3,6 +3,7 @@
[clojure.set :as set]
[clojure.test :refer [deftest is testing]]
[metabase-enterprise.dependencies.models.dependency :as deps.graph]
+ [metabase.graph.core :as graph]
[metabase.lib.core :as lib]
[metabase.queries.models.card :as card]
[metabase.test :as mt]
@@ -223,3 +224,71 @@
(let [card (card/create-card! (basic-orders) user)]
(is (= deps.graph/current-dependency-analysis-version
(t2/select-one-fn :dependency_analysis_version :model/Card :id (:id card))))))))))
+
+(deftest filtered-graph-dependencies-test
+ (testing "filtered-graph-dependencies respects filter clause"
+ (mt/with-temp [:model/User user {:email "[email protected]"}]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (let [{id1 :id :as card1} (card/create-card! (basic-orders) user)
+ {id2 :id :as card2} (card/create-card! (wrap-card card1) user)
+ {id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]
+ (testing "without filter, returns all dependencies"
+ (let [graph (deps.graph/graph-dependencies)
+ deps (graph/transitive graph [[:card id3]])]
+ (is (= #{[:card id2] [:card id1] [:table (mt/id :orders)]}
+ (set deps)))))
+ (testing "with filter excluding card1, omits card1 and its dependencies"
+ (let [filter-fn (fn [entity-type-field entity-id-field]
+ [:and
+ [:= entity-type-field "card"]
+ [:in entity-id-field [id2 id3]]])
+ graph (deps.graph/filtered-graph-dependencies filter-fn)
+ deps (graph/transitive graph [[:card id3]])]
+ (is (= #{[:card id2]}
+ (set deps))
+ "Should only include card2, not card1 or table")))
+ (testing "with filter excluding card2, breaks the chain"
+ (let [filter-fn (fn [entity-type-field entity-id-field]
+ [:and
+ [:= entity-type-field "card"]
+ [:in entity-id-field [id1 id3]]])
+ graph (deps.graph/filtered-graph-dependencies filter-fn)
+ deps (graph/transitive graph [[:card id3]])]
+ (is (= #{}
+ (set deps))
+ "Should be empty, chain is broken at card2")))))))))
+
+(deftest filtered-graph-dependents-test
+ (testing "filtered-graph-dependents respects filter clause"
+ (mt/with-temp [:model/User user {:email "[email protected]"}]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (let [{id1 :id :as card1} (card/create-card! (basic-orders) user)
+ {id2 :id :as card2} (card/create-card! (wrap-card card1) user)
+ {id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]
+ (testing "without filter, returns all dependents"
+ (let [graph (deps.graph/graph-dependents)
+ deps (graph/transitive graph [[:card id1]])]
+ (is (= #{[:card id2] [:card id3]}
+ (set deps)))))
+ (testing "with filter excluding card3, omits card3"
+ (let [filter-fn (fn [entity-type-field entity-id-field]
+ [:and
+ [:= entity-type-field "card"]
+ [:in entity-id-field [id1 id2]]])
+ graph (deps.graph/filtered-graph-dependents filter-fn)
+ deps (graph/transitive graph [[:card id1]])]
+ (is (= #{[:card id2]}
+ (set deps))
+ "Should only include card2, not card3")))
+ (testing "with filter excluding card2, breaks the chain"
+ (let [filter-fn (fn [entity-type-field entity-id-field]
+ [:and
+ [:= entity-type-field "card"]
+ [:in entity-id-field [id1 id3]]])
+ graph (deps.graph/filtered-graph-dependents filter-fn)
+ deps (graph/transitive graph [[:card id1]])]
+ (is (= #{}
+ (set deps))
+ "Should be empty, chain is broken at card2")))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
index a5a478b7f6a..d9057c22680 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj
@@ -1225,8 +1225,9 @@
{:request-options {:headers {"x-metabase-session" crowberto-ai-token}}}
{:conversation_id conversation-id})
(update :structured_output (fn [output]
- (filter #(#{(:id t1) (:id t2) (:id t3)} (:id %))
- output)))))))))))
+ (->> output
+ (filter #(#{(:id t1) (:id t2) (:id t3)} (:id %)))
+ (sort-by :id))))))))))))
(deftest get-transform-test
(mt/with-premium-features #{:metabot-v3 :transforms}
diff --git a/frontend/src/metabase-types/api/dependencies.ts b/frontend/src/metabase-types/api/dependencies.ts
index b606c3b2c37..cce2010315b 100644
--- a/frontend/src/metabase-types/api/dependencies.ts
+++ b/frontend/src/metabase-types/api/dependencies.ts
@@ -1,3 +1,8 @@
+// IMPORTANT: The field selections in the *DependencyNodeData types below
+// MUST be kept in sync with the backend field selections in:
+// enterprise/backend/src/metabase_enterprise/dependencies/api.clj
+// (See entity-select-fields map)
+
import type { Card, CardType } from "./card";
import type { Dashboard } from "./dashboard";
import type { Document } from "./document";