Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

fix: improve search performance with tables (#67055)

Edward Pagetd2212003ef3c
diff --git a/dev/src/dev/search_perf.clj b/dev/src/dev/search_perf.clj
new file mode 100644
index 00000000000..2d3144940d4
--- /dev/null
+++ b/dev/src/dev/search_perf.clj
@@ -0,0 +1,511 @@
+(ns dev.search-perf
+ "Performance testing utilities for search with large numbers of tables,
+ users, and permission groups.
+
+ Example usage:
+ ;; Run full benchmark with default settings (10000 tables, 10 groups, 100 users)
+ (run-full-benchmark!)
+
+ ;; Run with custom settings
+ (run-full-benchmark! {:num-tables 10000
+ :num-groups 10
+ :num-users 100
+ :iterations 5})
+
+ ;; Manual setup for interactive testing
+ (def test-env (create-test-environment! {:num-tables 1000
+ :num-groups 5
+ :num-users 50}))
+ (timed-search! \"table\" :user-id (:user-id test-env))
+ (cleanup-test-environment! test-env)"
+ (:require
+ [clojure.string :as str]
+ [dev.add-load :as add-load]
+ [metabase.api.common :as api]
+ [metabase.permissions.core :as perms]
+ [metabase.permissions.util :as perms-util]
+ [metabase.request.core :as request]
+ [metabase.search.config :as search.config]
+ [metabase.search.engine :as search.engine]
+ [metabase.search.impl :as search.impl]
+ [metabase.search.ingestion :as search.ingestion]
+ [metabase.util :as u]
+ [metabase.util.random :as u.random]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+;;; -------------------------------------------- Setup Utilities: Database & Tables ------------------------------------
+
+(defn- generate-table-name
+ "Generate a table name with a prefix and index for identification."
+ [prefix idx]
+ (format "%s_table_%05d" prefix idx))
+
+(defn create-test-database!
+ "Create a test database for performance testing.
+ Returns the database ID."
+ []
+ (let [{:keys [db-id]} (add-load/from-script
+ [[:model/Database {:?/db-id :id}
+ {:name (str "Search Perf Test DB " (u.random/random-name))
+ :engine :h2
+ :details {}}]])]
+ (perms/set-database-permission! (perms/all-users-group) db-id :perms/view-data :blocked)
+ (perms/set-database-permission! (perms/all-users-group) db-id :perms/create-queries :no)
+ db-id))
+
+(defn create-tables-batch!
+ "Create a batch of tables for the given database.
+ Returns the number of tables created."
+ [db-id prefix start-idx cnt]
+ (let [script (vec
+ (for [i (range start-idx (+ start-idx cnt))]
+ [:model/Table {}
+ {:db_id db-id
+ :name (generate-table-name prefix i)
+ :description (format "Performance test table %d for search benchmarking" i)
+ :active true}]))]
+ (add-load/from-script script)
+ cnt))
+
+(defn create-test-database-with-tables!
+ "Create a test database with the specified number of tables.
+ Tables are created in batches for better performance.
+
+ Options:
+ :batch-size - Number of tables per batch (default 500)
+ :prefix - Prefix for table names (default \"perf_test\")
+
+ Returns a map with :db-id and :table-ids."
+ ([num-tables]
+ (create-test-database-with-tables! num-tables {}))
+ ([num-tables {:keys [batch-size prefix]
+ :or {batch-size 500
+ prefix "perf_test"}}]
+ (let [db-id (create-test-database!)
+ num-batches (Math/ceil (/ num-tables batch-size))]
+ (println (format "Creating %d tables in %d batches of %d..."
+ num-tables (int num-batches) batch-size))
+ (let [timer (u/start-timer)]
+ (doseq [batch-num (range (int num-batches))]
+ (let [start-idx (* batch-num batch-size)
+ batch-count (min batch-size (- num-tables start-idx))]
+ (create-tables-batch! db-id prefix start-idx batch-count)
+ (when (zero? (mod (inc batch-num) 10))
+ (println (format " Created %d/%d tables..." (+ start-idx batch-count) num-tables)))))
+ (let [elapsed-ms (u/since-ms timer)
+ ;; Get all table IDs for this database
+ table-ids (t2/select-pks-vec :model/Table :db_id db-id)]
+ (println (format "Created %d tables in %.2f seconds (%.1f tables/sec)"
+ num-tables
+ (/ elapsed-ms 1000.0)
+ (/ (* num-tables 1000.0) elapsed-ms)))
+ {:db-id db-id
+ :table-ids table-ids})))))
+
+;;; -------------------------------------------- Setup Utilities: Groups & Users ---------------------------------------
+
+(defn create-permission-groups!
+ "Create n permission groups for testing.
+ Returns a vector of group IDs."
+ [n run-id]
+ (println (format "Creating %d permission groups..." n))
+ (let [timer (u/start-timer)
+ prefix (str "Search Perf Group " run-id " ")
+ script (vec
+ (for [i (range n)]
+ [:model/PermissionsGroup {}
+ {:name (str prefix i)}]))
+ _ (add-load/from-script script)
+ ;; Query for groups with our name pattern
+ group-ids (t2/select-pks-vec :model/PermissionsGroup :name [:like (str prefix "%")])
+ elapsed-ms (u/since-ms timer)]
+ (println (format "Created %d groups in %.2f seconds" n (/ elapsed-ms 1000.0)))
+ group-ids))
+
+(defn create-users-with-memberships!
+ "Create n users and assign them to permission groups.
+ Each user is assigned to a subset of groups based on their index.
+ Users are also added to the All Users group automatically by the model.
+
+ Returns a vector of user IDs."
+ [n group-ids run-id]
+ (println (format "Creating %d users with group memberships..." n))
+ (let [timer (u/start-timer)
+ num-groups (count group-ids)
+ email-domain (str "perftest" run-id ".example.com")
+ ;; Create users first
+ user-script (vec
+ (for [i (range n)]
+ [:model/User {}
+ {:first_name (format "PerfUser%d" i)
+ :last_name "Test"
+ :email (format "user%d@%s" i email-domain)
+ :password "password123"}]))
+ _ (add-load/from-script user-script)
+ ;; Query for users by email pattern
+ user-ids (t2/select-pks-vec :model/User :email [:like (str "%@" email-domain)])]
+
+ ;; Create group memberships - assign each user to some groups
+ ;; User i gets assigned to groups where (mod i num-groups) matches certain patterns
+ (println " Assigning users to groups...")
+ (let [membership-script
+ (vec
+ (for [user-idx (range n)
+ group-idx (range num-groups)
+ ;; Assign user to ~30% of groups based on a deterministic pattern
+ :when (zero? (mod (+ user-idx group-idx) 3))]
+ [:model/PermissionsGroupMembership {}
+ {:user_id (nth user-ids user-idx)
+ :group_id (nth group-ids group-idx)}]))]
+ (when (seq membership-script)
+ (add-load/from-script membership-script)))
+
+ (let [elapsed-ms (u/since-ms timer)]
+ (println (format "Created %d users with memberships in %.2f seconds" n (/ elapsed-ms 1000.0))))
+ user-ids))
+
+(defn grant-table-permissions-to-groups!
+ "Grant table-level permissions to groups.
+ Each group gets permissions to a subset of tables based on deterministic assignment.
+ This creates a realistic permission scenario where different groups have access
+ to different tables.
+
+ Assignment strategy:
+ - Each group gets access to ~5% of tables
+ - Tables are assigned based on (mod table-idx 20) pattern with group offset
+ - This ensures overlapping but distinct permission sets"
+ [table-ids group-ids]
+ (println (format "Granting table-level permissions to %d groups for %d tables..."
+ (count group-ids) (count table-ids)))
+ (let [timer (u/start-timer)
+ num-groups (count group-ids)
+ ;; Build permission assignments
+ ;; Each group gets ~5% of tables (1 in 20) with offset based on group index
+ assignments (for [group-idx (range num-groups)
+ :let [group-id (nth group-ids group-idx)
+ ;; Each group gets ~5% of tables with some overlap
+ ;; Using mod 20 gives 5%, offset by group-idx for variation
+ assigned-tables (filterv
+ (fn [table-idx]
+ ;; Deterministic pattern: group gets every 20th table
+ ;; offset by group index to create variation between groups
+ (zero? (mod (+ table-idx group-idx) 20)))
+ (range (count table-ids)))]]
+ {:group-id group-id
+ :group-idx group-idx
+ :table-indices assigned-tables
+ :num-tables (count assigned-tables)})]
+
+ ;; Grant permissions - batch by group for efficiency
+ (doseq [{:keys [group-id table-indices]} assignments]
+ (let [table-perms-view (into {} (for [tidx table-indices]
+ [(nth table-ids tidx) :unrestricted]))
+ table-perms-query (into {} (for [tidx table-indices]
+ [(nth table-ids tidx) :query-builder]))]
+ ;; Use batch permission setting
+ (when (seq table-perms-view)
+ (perms/set-table-permissions! group-id :perms/view-data table-perms-view)
+ (perms/set-table-permissions! group-id :perms/create-queries table-perms-query))))
+
+ (let [elapsed-ms (u/since-ms timer)]
+ (println (format "Granted table permissions in %.2f seconds" (/ elapsed-ms 1000.0))))))
+
+;;; -------------------------------------------- Test Environment Management -------------------------------------------
+
+(defn create-test-environment!
+ "Create a complete test environment with database, tables, groups, users, and permissions.
+
+ Options:
+ :num-tables - Number of tables to create (default 10000)
+ :num-groups - Number of permission groups (default 10)
+ :num-users - Number of users (default 100)
+ :prefix - Table name prefix (default \"perf_test\")
+
+ Returns a map containing:
+ :db-id - Database ID
+ :table-ids - Vector of table IDs
+ :group-ids - Vector of group IDs
+ :user-ids - Vector of user IDs
+ :user-id - ID of a sample non-admin user for testing
+ :run-id - Unique identifier for this test run"
+ ([]
+ (create-test-environment! {}))
+ ([{:keys [num-tables num-groups num-users prefix]
+ :or {num-tables 10000
+ num-groups 10
+ num-users 100
+ prefix "perf_test"}}]
+ (println "\n=== Creating Test Environment ===")
+ (let [timer (u/start-timer)
+ run-id (u.random/random-name)
+ ;; Create database and tables
+ {:keys [db-id table-ids]} (create-test-database-with-tables! num-tables {:prefix prefix})
+ ;; Create permission groups
+ group-ids (create-permission-groups! num-groups run-id)
+ ;; Grant table-level permissions to groups
+ _ (grant-table-permissions-to-groups! table-ids group-ids)
+ ;; Create users with group memberships
+ user-ids (create-users-with-memberships! num-users group-ids run-id)
+ ;; Pick a user in the middle for testing
+ test-user-id (nth user-ids (quot num-users 2))
+ elapsed-ms (u/since-ms timer)]
+ (println (format "Test environment created in %.2f seconds" (/ elapsed-ms 1000.0)))
+ (println (format " Run ID: %s" run-id))
+ (println (format " Database: %d" db-id))
+ (println (format " Tables: %d" num-tables))
+ (println (format " Groups: %d" num-groups))
+ (println (format " Users: %d" num-users))
+ (println (format " Test user ID: %d" test-user-id))
+ {:db-id db-id
+ :table-ids table-ids
+ :group-ids group-ids
+ :user-ids user-ids
+ :user-id test-user-id
+ :run-id run-id
+ :num-tables num-tables})))
+
+(defn cleanup-test-environment!
+ "Delete all resources created by create-test-environment!"
+ [{:keys [db-id group-ids user-ids]}]
+ (println "\n=== Cleaning Up Test Environment ===")
+ (let [timer (u/start-timer)]
+ ;; Delete group memberships first (foreign key constraints)
+ (when (seq user-ids)
+ (println (format "Deleting memberships for %d users..." (count user-ids)))
+ (t2/delete! :model/PermissionsGroupMembership :user_id [:in user-ids] :group_id [:<> 1]))
+ ;; Delete users
+ (when (seq user-ids)
+ (println (format "Deleting %d users..." (count user-ids)))
+ (t2/delete! :model/User :id [:in user-ids]))
+ ;; Delete data permissions for groups
+ (when (seq group-ids)
+ (println (format "Deleting permissions for %d groups..." (count group-ids)))
+ (t2/delete! :model/DataPermissions :group_id [:in group-ids]))
+ ;; Delete groups
+ (when (seq group-ids)
+ (println (format "Deleting %d groups..." (count group-ids)))
+ (t2/delete! :model/PermissionsGroup :id [:in group-ids]))
+ ;; Delete tables and database
+ (when db-id
+ (let [table-count (t2/count :model/Table :db_id db-id)]
+ (println (format "Deleting database %d and %d tables..." db-id table-count))
+ (t2/delete! :model/Table :db_id db-id)
+ (t2/delete! :model/Database :id db-id)))
+ (let [elapsed-ms (u/since-ms timer)]
+ (println (format "Cleanup completed in %.2f seconds" (/ elapsed-ms 1000.0))))))
+
+;;; ------------------------------------------------ Search Utilities -----------------------------------------------
+
+(defn- search-context-for-user
+ "Build a search context for a specific user."
+ [user-id search-string models]
+ (request/with-current-user user-id
+ (search.impl/search-context
+ {:current-user-id api/*current-user-id*
+ :current-user-perms @api/*current-user-permissions-set*
+ :is-superuser? api/*is-superuser?*
+ :is-impersonated-user? (perms-util/impersonated-user?)
+ :is-sandboxed-user? (perms-util/impersonated-user?)
+ :archived false
+ :context :default
+ :search-string search-string
+ :models models
+ :model-ancestors? false})))
+
+(defn search!
+ "Execute a search and return the results.
+
+ Options:
+ :user-id - User ID to run search as (required)
+ :models - Models to search (default all models)"
+ [search-string & {:keys [user-id models]
+ :or {models search.config/all-models}}]
+ (assert user-id "user-id is required for search!")
+ (let [ctx (search-context-for-user user-id search-string models)]
+ (search.engine/results ctx)))
+
+(defn timed-search!
+ "Execute a search and return timing information along with result count.
+
+ Options:
+ :user-id - User ID to run search as (required)
+ :models - Models to search (default all models)
+
+ Returns {:elapsed-ms <time> :result-count <count>}"
+ [search-string & {:keys [user-id models]
+ :or {models search.config/all-models}}]
+ (assert user-id "user-id is required for timed-search!")
+ (let [timer (u/start-timer)
+ results (search! search-string :user-id user-id :models models)
+ elapsed (u/since-ms timer)]
+ {:elapsed-ms elapsed
+ :result-count (count results)}))
+
+;;; ------------------------------------------------ Benchmark Functions --------------------------------------------
+
+(defn run-search-benchmark!
+ "Run a search benchmark with the given search string.
+
+ Options:
+ :user-id - User ID to run search as (required)
+ :iterations - Number of search iterations (default 5)
+ :models - Models to search (default all models)
+ :warmup - Number of warmup iterations (default 2)
+
+ Returns a map with benchmark statistics."
+ ([search-string opts]
+ (let [{:keys [user-id iterations warmup models]
+ :or {iterations 5
+ warmup 2
+ models search.config/all-models}} opts]
+ (assert user-id "user-id is required for benchmark!")
+ (println (format "\nRunning search benchmark for: \"%s\" (user-id: %d)" search-string user-id))
+ (println (format " Warmup iterations: %d" warmup))
+ (println (format " Benchmark iterations: %d" iterations))
+
+ ;; Warmup
+ (when (pos? warmup)
+ (println " Warming up...")
+ (dotimes [_ warmup]
+ (search! search-string :user-id user-id :models models)))
+
+ ;; Benchmark
+ (println " Running benchmark...")
+ (let [results (vec (for [i (range iterations)]
+ (let [result (timed-search! search-string :user-id user-id :models models)]
+ (println (format " Iteration %d: %.2fms (%d results)"
+ (inc i)
+ (:elapsed-ms result)
+ (:result-count result)))
+ result)))
+ times (mapv :elapsed-ms results)
+ avg (/ (reduce + times) (count times))
+ min-t (apply min times)
+ max-t (apply max times)
+ sorted (sort times)
+ p50 (nth sorted (int (* 0.5 (count sorted))))
+ p95 (nth sorted (int (* 0.95 (dec (count sorted)))))]
+ (println "\n Results:")
+ (println (format " Average: %.2fms" avg))
+ (println (format " Min: %.2fms" min-t))
+ (println (format " Max: %.2fms" max-t))
+ (println (format " P50: %.2fms" p50))
+ (println (format " P95: %.2fms" p95))
+ {:search-string search-string
+ :user-id user-id
+ :iterations iterations
+ :result-count (:result-count (first results))
+ :avg-ms avg
+ :min-ms min-t
+ :max-ms max-t
+ :p50-ms p50
+ :p95-ms p95
+ :all-times-ms times}))))
+
+(defn run-full-benchmark!
+ "Run a full benchmark suite with multiple search queries.
+
+ Creates a test environment with database, tables, groups, and users,
+ runs benchmarks with various search patterns using a non-admin user,
+ and optionally cleans up the test data.
+
+ Options:
+ :num-tables - Number of tables to create (default 10000)
+ :num-groups - Number of permission groups (default 10)
+ :num-users - Number of users (default 100)
+ :iterations - Benchmark iterations per query (default 5)
+ :cleanup? - Whether to cleanup after (default true)
+ :prefix - Table name prefix (default \"perf_test\")
+
+ Returns benchmark results for all queries."
+ ([]
+ (run-full-benchmark! {}))
+ ([{:keys [num-tables num-groups num-users iterations cleanup? prefix]
+ :or {num-tables 10000
+ num-groups 10
+ num-users 100
+ iterations 5
+ cleanup? true
+ prefix "perf_test"}}]
+ (println (format "\n%s" (str/join (repeat 60 "="))))
+ (println "SEARCH PERFORMANCE BENCHMARK (Non-Admin User)")
+ (println (str/join (repeat 60 "=")))
+ (println "\nConfiguration:")
+ (println (format " Tables: %d" num-tables))
+ (println (format " Permission Groups: %d" num-groups))
+ (println (format " Users: %d" num-users))
+ (println (format " Iterations per query: %d" iterations))
+
+ (let [test-env (create-test-environment! {:num-tables num-tables
+ :num-groups num-groups
+ :num-users num-users
+ :prefix prefix})
+ user-id (:user-id test-env)
+ ;; Wait for search index to update if using async indexing
+ _ (binding [search.ingestion/*force-sync* true]
+ (Thread/sleep 1000))
+ queries [;; Exact prefix match
+ (str prefix "_table_00001")
+ ;; Partial prefix match
+ (str prefix "_table")
+ ;; Generic term that should match descriptions
+ "performance test"
+ ;; Broad search
+ "table"
+ ;; Very specific search
+ (str prefix "_table_05000")]]
+ (try
+ (let [results (vec (for [query queries]
+ (run-search-benchmark! query {:user-id user-id
+ :iterations iterations})))]
+ (println (format "\n%s" (str/join (repeat 60 "="))))
+ (println "SUMMARY")
+ (println (str/join (repeat 60 "=")))
+ (doseq [{:keys [search-string avg-ms result-count]} results]
+ (println (format " \"%s\": %.2fms avg, %d results"
+ search-string avg-ms result-count)))
+ {:results results
+ :test-env test-env})
+ (finally
+ (when cleanup?
+ (cleanup-test-environment! test-env)))))))
+
+(comment
+ ;; Example usage:
+
+ ;; Quick test with fewer tables/users
+ (run-full-benchmark! {:num-tables 1
+ :num-groups 5
+ :num-users 4
+ :iterations 1})
+
+ ;; Full benchmark with 10,000 tables
+ (run-full-benchmark! {:num-tables 10000
+ :num-groups 25
+ :num-users 100
+ :iterations 10})
+
+ ;; {:p95-ms 539.613833, :result-count 10000, :avg-ms 536.6528708, :min-ms 529.310541, :all-times-ms [548.471834 534.8465 538.846875 536.401583 535.707 ...], ...}
+ ;; {:p95-ms 366.195458, :result-count 10000, :avg-ms 360.5523999, :min-ms 352.951208, :all-times-ms [355.839542 366.195458 352.951208 356.9155 362.09225 ...], ...}
+ ;; {:p95-ms 110.475333, :result-count 4000, :avg-ms 108.0612666, :min-ms 105.713041, :all-times-ms [112.747333 108.450292 107.548208 105.93625 108.159375 ...], ...}
+
+ ;; Manual setup for interactive testing
+ (def test-env (create-test-environment! {:num-tables 1000
+ :num-groups 35
+ :num-users 200}))
+
+ ;; Run individual searches with the non-admin user
+ (e/explain
+ (timed-search! "perf_test_table" :user-id (:user-id test-env)))
+ (timed-search! "table" :user-id (:user-id test-env) :models #{"table"})
+
+ (require '[dev.explain-analyze :as e])
+ ;; Try with different users
+ (e/explain
+ (timed-search! "table" :user-id (first (:user-ids test-env))))
+ (timed-search! "table" :user-id (last (:user-ids test-env)))
+
+ ;; Cleanup
+ (cleanup-test-environment! test-env))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
index 4db0d87304b..cf5f04dae83 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
@@ -228,7 +228,7 @@
- Collection-based (:model/Card, :model/Dashboard, :model/Document, :model/NativeQuerySnippet):
Uses collection/visible-collection-filter-clause for collection filtering and adds archived entity filtering.
Native query snippets have additional restrictions for sandboxed users.
- - Table: Uses mi/visible-filter-clause with appropriate permissions and filters by active/visibility_type.
+ - Table: Uses perms/visible-table-filter-select with appropriate permissions and filters by active/visibility_type.
Follows search API conventions: active=true AND visibility_type=nil for non-archived tables.
Note: archived_at is not checked separately as archived tables always have active=false."
([entity-type-field entity-id-field]
@@ -279,13 +279,13 @@
[:in entity-id-field {:select [:id]
:from [table-name]
:where [:and
- (mi/visible-filter-clause
- model
- id-column
- {:user-id api/*current-user-id*
- :is-superuser? api/*is-superuser?*}
- {:perms/view-data :unrestricted
- :perms/create-queries :query-builder})
+ [:in id-column
+ (perms/visible-table-filter-select
+ :id
+ {:user-id api/*current-user-id*
+ :is-superuser? api/*is-superuser?*}
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder})]
(case include-archived-items
:exclude [:and
[:= active-column true]
@@ -305,13 +305,16 @@
[:in table-id-column
{:select [:metabase_table.id]
:from [:metabase_table]
- :where (mi/visible-filter-clause
- :model/Table
- :metabase_table.id
- {:user-id api/*current-user-id*
- :is-superuser? api/*is-superuser?*}
- {:perms/view-data :unrestricted
- :perms/create-queries :query-builder})}]
+ ;; using this clause because we had to change the mi/visible-filter-clause
+ ;; to allow returning CTE based filters
+ ;; TODO(ed 2025-12-16: support using CTES in filters in dependency graph)
+ :where [:in :metabase_table.id
+ (perms/visible-table-filter-select
+ :id
+ {:user-id api/*current-user-id*
+ :is-superuser? api/*is-superuser?*}
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder})]}]
;; Filter by archived status
(case include-archived-items
:exclude [:= archived-column false]
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/table_utils.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/table_utils.clj
index 35316695c44..36a1dcdb2f2 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/table_utils.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/table_utils.clj
@@ -44,19 +44,21 @@
priority-tables []
exclude-table-ids #{}}}]
(let [priority-table-ids (set (map :id priority-tables))
+ {table-where-clause :clause table-cte :with} (mi/visible-filter-clause :model/Table
+ :id
+ {:user-id api/*current-user-id*
+ :is-superuser? api/*is-superuser?*}
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder-and-native})
;; Fetch most viewed tables, excluding priority tables and excluded tables
fill-tables (t2/select [:model/Table :id :db_id :name :schema :description]
- :db_id database-id
- :active true
+ :db_id database-id
+ :active true
:visibility_type nil
- {:where (mi/visible-filter-clause :model/Table
- :id
- {:user-id api/*current-user-id*
- :is-superuser? api/*is-superuser?*}
- {:perms/view-data :unrestricted
- :perms/create-queries :query-builder-and-native})
- :order-by [[:view_count :desc]]
- :limit all-tables-limit})
+ (cond-> {:where table-where-clause
+ :order-by [[:view_count :desc]]
+ :limit all-tables-limit}
+ table-cte (assoc :with table-cte)))
fill-tables (remove #(or (priority-table-ids (:id %))
(exclude-table-ids (:id %))) fill-tables)
fill-tables (t2/hydrate fill-tables :fields)
@@ -83,19 +85,21 @@
priority-tables []
exclude-table-ids #{}}}]
(let [priority-table-ids (set (map :id priority-tables))
+ {table-where-clause :clause table-cte :with} (mi/visible-filter-clause :model/Table
+ :id
+ {:user-id api/*current-user-id*
+ :is-superuser? api/*is-superuser?*}
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder-and-native})
;; Fetch most viewed tables, excluding priority tables and excluded tables
fill-tables (t2/select [:model/Table :id :db_id :name :schema :description]
:db_id database-id
:active true
:visibility_type nil
- {:where (mi/visible-filter-clause :model/Table
- :id
- {:user-id api/*current-user-id*
- :is-superuser? api/*is-superuser?*}
- {:perms/view-data :unrestricted
- :perms/create-queries :query-builder-and-native})
- :order-by [[:view_count :desc]]
- :limit all-tables-limit})
+ (cond-> {:where table-where-clause
+ :order-by [[:view_count :desc]]
+ :limit all-tables-limit}
+ table-cte (assoc :with table-cte)))
fill-tables (remove #(or (priority-table-ids (:id %))
(exclude-table-ids (:id %))) fill-tables)
all-tables (concat priority-tables fill-tables)
@@ -164,12 +168,14 @@
(defn- visible-filter-clause
[]
- (mi/visible-filter-clause :model/Table
- :id
- {:user-id api/*current-user-id*
- :is-superuser? api/*is-superuser?*}
- {:perms/view-data :unrestricted
- :perms/create-queries :query-builder-and-native}))
+ (let [{table-where-clause :clause table-cte :with} (mi/visible-filter-clause :model/Table
+ :id
+ {:user-id api/*current-user-id*
+ :is-superuser? api/*is-superuser?*}
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder-and-native})]
+ (cond-> {:where table-where-clause}
+ table-cte (assoc :with table-cte))))
(defn find-matching-tables
"Find tables in the database that are similar to the unrecognized tables using fuzzy matching.
@@ -186,6 +192,7 @@
Returns:
A vector of realized Table model instances that match the unrecognized tables."
[database-id unrecognized-tables used-ids]
+
(into []
(keep (fn [table]
(when (some #(matching-tables? table % {:match-schema? false}) unrecognized-tables)
@@ -194,8 +201,8 @@
:db_id database-id
:active true
:visibility_type nil
- (cond-> {:where (visible-filter-clause)
- :limit 10000}
+ (cond-> (assoc (visible-filter-clause)
+ :limit 10000)
(seq used-ids) (update :where #(if %
[:and % [:not-in :id used-ids]]
[:not-in :id used-ids]))))))
@@ -212,7 +219,7 @@
:id [:in table-ids]
:active true
:visibility_type nil
- {:where (visible-filter-clause)})))
+ (visible-filter-clause))))
(defn used-tables
"Return all tables used in the query, including fuzzy-matched ones.
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
index 5292b45c93d..9004220c1c7 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj
@@ -568,6 +568,24 @@
(update base-query :where #(into [:and] [% filters]))
base-query)))
+(defn- flatten-ctes
+ "Flatten nested :with clauses into a single top-level :with.
+ PostgreSQL doesn't support nested CTEs, so we need to extract all nested :with clauses
+ and put them at the top level.
+
+ Takes a query map and returns {:ctes [...] :query query-without-with}"
+ [query]
+ (if-not (:with query)
+ {:ctes [] :query query}
+ (let [ctes (reduce
+ (fn [acc [cte-name cte-query]]
+ (let [{:keys [ctes query]} (flatten-ctes cte-query)]
+ (into acc (conj ctes [cte-name query]))))
+ []
+ (:with query))]
+ {:ctes ctes
+ :query (dissoc query :with)})))
+
(defn- hybrid-select
"For a given `col-name` return a :coalesce expression to reference it from the outer hybrid search query.
@@ -600,8 +618,14 @@
;; The purpose of this query is just to project the coalesced hybrid columns with standard names so the scorers know
;; what to call them (e.g. :model rather than [:coalesced :v.model :t.model]). Likewise, the :search_index alias
;; allows us to re-use scoring expressions between the appdb and semantic backends without adjusting column names.
+ ;;
+ ;; We flatten the nested CTEs because PostgreSQL doesn't support nested WITH clauses.
+ ;; The hybrid-query contains nested :with clauses from semantic-search-query and keyword-search-query,
+ ;; which we extract and place at the top level.
(let [hybrid-query (hybrid-search-query index embedding search-context)
- full-query {:with [[:hybrid_results hybrid-query]]
+ {:keys [ctes query]} (flatten-ctes hybrid-query)
+ all-ctes (conj ctes [:hybrid_results query])
+ full-query {:with all-ctes
:select [:id :model_id :model :content :verified :metadata :semantic_rank :keyword_rank]
:from [[:hybrid_results :search_index]]
:limit (semantic-settings/semantic-search-results-limit)}]
diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.clj
index 46d32a1b309..c9ae08b0d8a 100644
--- a/src/metabase/models/interface.clj
+++ b/src/metabase/models/interface.clj
@@ -674,10 +674,12 @@
"Please consider adding one. See dox for `can-update?` for more details."))))
(defmulti visible-filter-clause
- "Return a honey SQL query fragment that will limit another query to only selecting records visible to the supplied user
- by filtering on a supplied column or honeysql expression, using a the map of permission type->minimum permission-level.
+ "Return a map with:
+ - :clause - honey SQL WHERE clause fragment to filter records visible to the user
+ - :with - optional vector of CTE definitions [[name query] ...] to be merged into the query
- Defaults to returning a no-op false statement 0=1."
+ Uses the map of permission type->minimum permission-level for filtering.
+ Defaults to returning a no-op false statement {:clause [:= 0 1]}."
{:arglists '([model column-or-exp user-info perm-type->perm-level])}
dispatch-on-model)
@@ -768,7 +770,7 @@
(defmethod visible-filter-clause :default
[_m _column-or-expression _user-info _perm-type->perm-level]
- [:= [:inline 0] [:inline 1]])
+ {:clause [:= [:inline 0] [:inline 1]]})
;;;; [[to-json]]
diff --git a/src/metabase/permissions/core.clj b/src/metabase/permissions/core.clj
index 90fe3312e2c..574e84ab63c 100644
--- a/src/metabase/permissions/core.clj
+++ b/src/metabase/permissions/core.clj
@@ -67,6 +67,7 @@
PermissionMapping
visible-database-filter-select
visible-table-filter-select
+ visible-table-filter-with-cte
select-tables-and-groups-granting-perm]
[metabase.permissions.models.permissions
audit-namespace-clause
diff --git a/src/metabase/permissions/models/data_permissions/sql.clj b/src/metabase/permissions/models/data_permissions/sql.clj
index f8b50187d0f..726e6cf2b41 100644
--- a/src/metabase/permissions/models/data_permissions/sql.clj
+++ b/src/metabase/permissions/models/data_permissions/sql.clj
@@ -75,17 +75,13 @@
:least (perm-type-to-least-int-case perm-type :dp.perm_value))
(perm-type-to-int-inline perm-type required-level)])
-(mu/defn- user-in-group-half-join
+(mu/defn- user-in-group-clause
+ "Returns a simple IN clause to check if dp.group_id is in the user's groups.
+ This is more efficient than nested EXISTS subqueries."
[user-id :- pos-int?]
- [:exists {:select [1]
- :from [[:permissions_group :pg]]
- :where [:and
- [:= :pg.id :dp.group_id]
- [:exists {:select [1]
- :from [[:permissions_group_membership :pgm]]
- :where [:and
- [:= :pgm.group_id :pg.id]
- [:= :pgm.user_id [:inline user-id]]]}]]}])
+ [:in :dp.group_id {:select [:group_id]
+ :from [:permissions_group_membership]
+ :where [:= :user_id [:inline user-id]]}])
(mu/defn- has-perms-for-table-as-honey-sql?
"Builds an EXIST (SELECT ...) half-join to filter tables that a user has the required permissions for. It builds the subselect by as a
@@ -103,7 +99,7 @@
[:and [:= :mt.db_id :dp.db_id]
[:= :dp.table_id nil]]
[:= :mt.id :dp.table_id]]
- (user-in-group-half-join user-id)]
+ (user-in-group-clause user-id)]
:group-by [:mt.id]
:having [(perm-condition perm-type required-level (or most-or-least :least))]}])
@@ -139,6 +135,91 @@
(not (sequential? perm-level)) vector))]))
permission-mapping))})
+(mu/defn- permission-type-having-clause
+ "Builds a HAVING clause condition for a single permission type using conditional aggregation."
+ [perm-type :- :keyword
+ required-level :- :keyword
+ most-or-least :- [:enum :most :least]]
+ (let [agg-fn (case most-or-least
+ :most :max
+ :least :min)
+ ;; Build conditional: CASE WHEN dp.perm_type = ? THEN <int-case> END
+ conditional-case [:case
+ [:= :dp.perm_type (h2x/literal perm-type)]
+ (perm-type-to-int-case perm-type :dp.perm_value)]]
+ [(case most-or-least
+ :most :>=
+ :least :<=)
+ (if (= perm-type :perms/view-data)
+ ;; Special handling for view-data permission priority
+ (let [minimum-perm-value [agg-fn
+ [:case
+ [:= :dp.perm_type (h2x/literal perm-type)]
+ [:case
+ [:= :dp.perm_value [:inline "unrestricted"]] [:inline 0]
+ [:= :dp.perm_value [:inline "blocked"]] [:inline 1]
+ [:= :dp.perm_value [:inline "legacy-no-self-service"]] [:inline 2]]]]]
+ [:case
+ [:= minimum-perm-value [:inline 0]] [:inline 0]
+ [:= minimum-perm-value [:inline 1]] [:inline 2]
+ [:= minimum-perm-value [:inline 2]] [:inline 1]])
+ [agg-fn conditional-case])
+ (perm-type-to-int-inline perm-type required-level)]))
+
+(mu/defn visible-table-filter-with-cte
+ "Returns a map with :with (CTE definitions) and :clause (WHERE clause fragment) for filtering
+ tables visible to the user. Uses a CTE to compute permitted table IDs once rather than using
+ correlated subqueries, which provides better performance for large numbers of tables.
+
+ The returned map can be merged into a query by adding :with to the query's :with vector
+ and using :clause in the WHERE clause.
+
+ Uses UNION ALL to separate table-level and database-level permission lookups, avoiding
+ inefficient BitmapOr scans that occur with OR joins."
+ [column-or-exp :- :any
+ {:keys [user-id is-superuser?]} :- UserInfo
+ permission-mapping :- PermissionMapping]
+ (if is-superuser?
+ {:clause [:= [:inline 1] [:inline 1]]}
+ (let [perm-types (keys permission-mapping)
+ perm-type-filter (into [:or]
+ (map (fn [pt] [:= :dp.perm_type (h2x/literal pt)])
+ perm-types))
+ having-conditions (into [:and]
+ (map (fn [[perm-type perm-level]]
+ (let [[level most-or-least] (if (sequential? perm-level)
+ perm-level
+ [perm-level :least])]
+ (permission-type-having-clause perm-type level most-or-least)))
+ permission-mapping))
+ user-groups-clause (user-in-group-clause user-id)]
+ {:with [;; First CTE: collect all permission grants that apply to each table
+ [:table_permissions
+ {:union-all
+ [;; Table-level permissions (direct grant to table)
+ {:select [:mt.id :dp.perm_type :dp.perm_value]
+ :from [[:data_permissions :dp]]
+ :join [[:metabase_table :mt] [:= :mt.id :dp.table_id]]
+ :where [:and
+ [:not= :dp.table_id nil]
+ user-groups-clause
+ perm-type-filter]}
+ ;; Database-level permissions (applies to all tables in db)
+ {:select [:mt.id :dp.perm_type :dp.perm_value]
+ :from [[:data_permissions :dp]]
+ :join [[:metabase_table :mt] [:= :mt.db_id :dp.db_id]]
+ :where [:and
+ [:= :dp.table_id nil]
+ user-groups-clause
+ perm-type-filter]}]}]
+ ;; Second CTE: aggregate and filter by permission requirements
+ [:permitted_tables
+ {:select [:dp.id]
+ :from [[:table_permissions :dp]]
+ :group-by [:dp.id]
+ :having having-conditions}]]
+ :clause [:in column-or-exp {:select [:id] :from [:permitted_tables]}]})))
+
(mu/defn select-tables-and-groups-granting-perm
"Selects table.id and the group.id of all permissions groups that give the provided user the provided permission level or a
permission level either more or less restrictive than the supplied level."
@@ -182,7 +263,7 @@
:where [:and
[:= :dp.perm_type (h2x/literal perm-type)]
[:= :md.id :dp.db_id]
- (user-in-group-half-join user-id)]
+ (user-in-group-clause user-id)]
:group-by [:md.id]
:having [(perm-condition perm-type required-level (or most-or-least :least))]}])
diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.clj
index 6ad0ffd9a26..12685ebd5aa 100644
--- a/src/metabase/search/appdb/core.clj
+++ b/src/metabase/search/appdb/core.clj
@@ -75,20 +75,22 @@
(update :last_edited_at parse-datetime)))
(defn add-table-where-clauses
- "Add a `WHERE` clause to the query to only return tables the current user has access to"
+ "Add a `WHERE` clause to the query to only return tables the current user has access to.
+ Also adds any CTEs required for permission filtering."
[search-ctx qry]
- (sql.helpers/where qry
- [:or
- [:= :search_index.model nil]
- [:!= :search_index.model [:inline "table"]]
- [:and
- [:= :search_index.model [:inline "table"]]
- [:exists {:select [1]
- :from [[:metabase_table :mt_toplevel]]
- :where [:and [:= :mt_toplevel.id [:cast :search_index.model_id (case (mdb/db-type)
- :mysql :signed
- :integer)]]
- (search.permissions/permitted-tables-clause search-ctx :mt_toplevel.id)]}]]]))
+ (let [model-id-col [:cast :search_index.model_id (case (mdb/db-type)
+ :mysql :signed
+ :integer)]
+ {:keys [with clause]} (search.permissions/permitted-tables-clause search-ctx model-id-col)]
+ (cond-> qry
+ (seq with) (update :with (fnil into []) with)
+ true (sql.helpers/where
+ [:or
+ [:= :search_index.model nil]
+ [:!= :search_index.model [:inline "table"]]
+ [:and
+ [:= :search_index.model [:inline "table"]]
+ clause]]))))
(def null-collection-id-should-be-ignored-for-unpublished-tables-clause
"For every model except tables, a null collection ID means the thing is in Our Analytics.
diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.clj
index 91352cfa012..16bb55e05e3 100644
--- a/src/metabase/search/in_place/legacy.clj
+++ b/src/metabase/search/in_place/legacy.clj
@@ -169,20 +169,26 @@
query))
(defn add-table-where-clauses
- "Add a `WHERE` clause to the query to only return tables the current user has access to"
+ "Add a `WHERE` clause to the query to only return tables the current user has access to.
+ Also adds any CTEs required for permission filtering."
[qry model search-ctx]
- (sql.helpers/where qry (case model
- "table" [:and
- (search.permissions/permitted-tables-clause search-ctx :table.id)
- [:or
- [:not :is_published]
- (search.permissions/permitted-collections-clause search-ctx :collection_id)]]
- "search-index" [:or
- [:= :search_index.model nil]
- [:!= :search_index.model [:inline "table"]]
- [:and
- [:= :search_index.model [:inline "table"]]
- (search.permissions/permitted-tables-clause search-ctx :search_index.model_id)]])))
+ (let [col (case model "table" :table.id "search-index" :search_index.model_id)
+ {:keys [with clause]} (search.permissions/permitted-tables-clause search-ctx col)]
+ (cond-> qry
+ (seq with) (update :with (fnil into []) with)
+ true (sql.helpers/where
+ (case model
+ "table" [:and
+ clause
+ [:or
+ [:not :is_published]
+ (search.permissions/permitted-collections-clause search-ctx :collection_id)]]
+ "search-index" [:or
+ [:= :search_index.model nil]
+ [:!= :search_index.model [:inline "table"]]
+ [:and
+ [:= :search_index.model [:inline "table"]]
+ clause]])))))
(mu/defn add-collection-join-and-where-clauses
"Add a `WHERE` clause to the query to only return Collections the Current User has access to; join against Collection,
@@ -619,15 +625,31 @@
(sql.helpers/left-join [:collection :collection] [:and :table.is_published [:= :table.collection_id :collection.id]])
(sql.helpers/left-join :metabase_database [:= :table.db_id :metabase_database.id]))))
+(defn- extract-and-hoist-ctes
+ "Extract :with clauses from a collection of queries and return a map with:
+ - :ctes - all CTEs collected from queries (deduplicated by name)
+ - :queries - queries with their :with clauses removed
+
+ This is needed because MySQL/MariaDB doesn't support CTEs inside UNION ALL subqueries -
+ the WITH clause must be at the outermost level of the statement."
+ [queries]
+ (let [all-ctes (into [] (comp (mapcat :with) (distinct)) queries)
+ queries-without-ctes (mapv #(dissoc % :with) queries)]
+ {:ctes all-ctes
+ :queries queries-without-ctes}))
+
(defmethod search.engine/model-set :search.engine/in-place
[search-ctx]
- (let [model-queries (for [model (search.in-place.filter/search-context->applicable-models
- ;; It's unclear why we don't use the existing :models
- (assoc search-ctx :models search.config/all-models))]
- {:nest (sql.helpers/limit (search-query-for-model model search-ctx) 1)})
- query (when (pos-int? (count model-queries))
- {:select [:*]
- :from [[{:union-all model-queries} :dummy_alias]]})]
+ (let [raw-queries (vec (for [model (search.in-place.filter/search-context->applicable-models
+ ;; It's unclear why we don't use the existing :models
+ (assoc search-ctx :models search.config/all-models))]
+ (search-query-for-model model search-ctx)))
+ {:keys [ctes queries]} (extract-and-hoist-ctes raw-queries)
+ nested-queries (mapv #(hash-map :nest (sql.helpers/limit % 1)) queries)
+ query (when (pos-int? (count nested-queries))
+ (cond-> {:select [:*]
+ :from [[{:union-all nested-queries} :dummy_alias]]}
+ (seq ctes) (assoc :with ctes)))]
(into #{} (map :model) (some-> query mdb/query))))
(mu/defn full-search-query
@@ -645,13 +667,16 @@
{:limit search.config/*db-max-results*})
:else
- {:select [:*]
- :from [[{:union-all (vec (for [model models
- :let [query (search-query-for-model model search-ctx)]
- :when (seq query)]
- query))} :alias_is_required_by_sql_but_not_needed_here]]
- :order-by order-clause
- :limit search.config/*db-max-results*})))
+ (let [model-queries (vec (for [model models
+ :let [query (search-query-for-model model search-ctx)]
+ :when (seq query)]
+ query))
+ {:keys [ctes queries]} (extract-and-hoist-ctes model-queries)]
+ (cond-> {:select [:*]
+ :from [[{:union-all queries} :alias_is_required_by_sql_but_not_needed_here]]
+ :order-by order-clause
+ :limit search.config/*db-max-results*}
+ (seq ctes) (assoc :with ctes))))))
;; Return a reducible-query corresponding to searching the entities without an index.
(defn- results
diff --git a/src/metabase/search/permissions.clj b/src/metabase/search/permissions.clj
index fb293ef0040..de8b85fe044 100644
--- a/src/metabase/search/permissions.clj
+++ b/src/metabase/search/permissions.clj
@@ -49,8 +49,9 @@
[:= :collection.namespace "shared-tenant-collection"]]])
(mu/defn permitted-tables-clause
- "Build the WHERE clause corresponding to which tables the given user has access to."
- [{:keys [current-user-id is-superuser?]} :- SearchContext table-id-col :- :keyword]
+ "Build the WHERE clause and optional CTEs for table permission filtering.
+ Returns a map with :clause (WHERE clause fragment) and :with (optional CTE definitions)."
+ [{:keys [current-user-id is-superuser?]} :- SearchContext table-id-col :- [:or :keyword [:vector :keyword]]]
(mi/visible-filter-clause
:model/Table
table-id-col
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index 7eff6dbd0ca..51df676ef32 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -274,8 +274,7 @@
column-or-exp :- :any
user-info :- perms/UserInfo
permission-mapping :- perms/PermissionMapping]
- [:in column-or-exp
- (perms/visible-table-filter-select :id user-info permission-mapping)])
+ (perms/visible-table-filter-with-cte column-or-exp user-info permission-mapping))
;;; ------------------------------------------------ Serdes Hashing -------------------------------------------------
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 3e3ea5573f4..4eba736e07f 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -129,8 +129,8 @@
(mu/defmethod mi/visible-filter-clause :model/Database
[_model column-or-exp user-info permission-mapping]
- [:in column-or-exp
- (perms/visible-database-filter-select user-info permission-mapping)])
+ {:clause [:in column-or-exp
+ (perms/visible-database-filter-select user-info permission-mapping)]})
(defn- infer-db-schedules
"Infer database schedule settings based on its options."
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.clj
index 0cdf7ef2534..8bcdd6d18ae 100644
--- a/src/metabase/warehouses_rest/api.clj
+++ b/src/metabase/warehouses_rest/api.clj
@@ -274,7 +274,7 @@
[:= :router_database_id router-database-id]
[:= :router_database_id nil])]
where-clause (if filter-by-data-access?
- [:and base-where (mi/visible-filter-clause :model/Database :id user-info permission-mapping)]
+ [:and base-where (:clause (mi/visible-filter-clause :model/Database :id user-info permission-mapping))]
base-where)
dbs (t2/select :model/Database {:order-by [:%lower.name :%lower.engine]
:where where-clause})]
diff --git a/test/metabase/warehouse_schema/models/table_test.clj b/test/metabase/warehouse_schema/models/table_test.clj
index 9714a21c249..8f2badd053a 100644
--- a/test/metabase/warehouse_schema/models/table_test.clj
+++ b/test/metabase/warehouse_schema/models/table_test.clj
@@ -180,191 +180,316 @@
(t2/hydrate :pk_field)
:pk_field))))
-(deftest visible-filter-clause-table-level-test
- (testing "visible-tables-filter-clause generates a HoneySQL clause that filters tables based on user permissions"
+;;; ------------------------------------------ visible-filter-clause tests ------------------------------------------
+
+(defn- fetch-visible-table-ids
+ "Fetches visible table IDs using `visible-filter-clause` with the new `:clause/:with` return structure."
+ [db-id user-info permission-mapping table-id-field]
+ (let [{:keys [clause with]} (mi/visible-filter-clause :model/Table table-id-field user-info permission-mapping)]
+ (t2/select-pks-set [:model/Table]
+ (cond-> {:where [:and [:= :db_id db-id] clause]}
+ with (assoc :with with)))))
+
+(defn- superuser-info
+ "Returns user-info map for a superuser."
+ [user-id]
+ {:user-id user-id
+ :is-superuser? true})
+
+(defn- regular-user-info
+ "Returns user-info map for a non-superuser."
+ [user-id]
+ {:user-id user-id
+ :is-superuser? false})
+
+(deftest visible-filter-clause-superuser-sees-all-tables-test
+ (testing "Superuser should see all tables regardless of permissions"
(mt/with-no-data-perms-for-all-users!
- (mt/with-temp [:model/Database {db-id :id} {}
- :model/Table {table-view-data-unrestricted-create-queries-query-builder-id :id} {:db_id db-id :name "Table1"}
- :model/Table {table-view-data-unrestricted-create-queries-query-builder-and-native-id :id} {:db_id db-id :name "Table2"}
- :model/Table {table-view-data-blocked-create-queries-no-id :id} {:db_id db-id :name "Table3"}
- :model/Table {table-view-data-legacy-no-self-service-create-queries-no-id :id} {:db_id db-id :name "Table4"}
- :model/PermissionsGroup pg1 {}
- :model/PermissionsGroup pg2 {}]
-
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}
+ :model/Table {table-3 :id} {:db_id db-id :name "Table3"}
+ :model/PermissionsGroup pg {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg)
+ (t2/delete! :model/DataPermissions :db_id db-id)
+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)
+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)
+ (is (= #{table-1 table-2 table-3}
+ (fetch-visible-table-ids db-id
+ (superuser-info (mt/user->id :rasta))
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder}
+ :id)))))))
+
+(deftest visible-filter-clause-filters-by-view-and-query-perms-test
+ (testing "Non-superuser sees only tables where they have both view and query permissions"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}
+ :model/Table {table-blocked :id} {:db_id db-id :name "Table3"}
+ :model/PermissionsGroup pg {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg)
+ (t2/delete! :model/DataPermissions :db_id db-id)
+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)
+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)
+ (data-perms/set-table-permission! pg table-1 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg table-1 :perms/create-queries :query-builder)
+ (data-perms/set-table-permission! pg table-2 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg table-2 :perms/create-queries :query-builder-and-native)
+ (data-perms/set-table-permission! pg table-blocked :perms/view-data :blocked)
+ (data-perms/set-table-permission! pg table-blocked :perms/create-queries :no)
+ (is (= #{table-1 table-2}
+ (fetch-visible-table-ids db-id
+ (regular-user-info (mt/user->id :rasta))
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder}
+ :id)))))))
+
+(deftest visible-filter-clause-coalesce-expression-test
+ (testing "Filter clause works with coalesce expression for table ID"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}
+ :model/PermissionsGroup pg {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg)
+ (t2/delete! :model/DataPermissions :db_id db-id)
+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)
+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)
+ (data-perms/set-table-permission! pg table-1 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg table-1 :perms/create-queries :query-builder)
+ (data-perms/set-table-permission! pg table-2 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg table-2 :perms/create-queries :query-builder-and-native)
+ (is (= #{table-1 table-2}
+ (fetch-visible-table-ids db-id
+ (regular-user-info (mt/user->id :rasta))
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder}
+ [:coalesce :id :metabase_table.id])))))))
+
+(deftest visible-filter-clause-qualified-keyword-test
+ (testing "Filter clause works with qualified keyword for table ID"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}
+ :model/PermissionsGroup pg {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg)
+ (t2/delete! :model/DataPermissions :db_id db-id)
+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)
+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)
+ (data-perms/set-table-permission! pg table-1 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg table-1 :perms/create-queries :query-builder)
+ (data-perms/set-table-permission! pg table-2 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg table-2 :perms/create-queries :query-builder-and-native)
+ (is (= #{table-1 table-2}
+ (fetch-visible-table-ids db-id
+ (regular-user-info (mt/user->id :rasta))
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder}
+ :metabase_table.id)))))))
+
+(deftest visible-filter-clause-view-data-only-test
+ (testing "Non-superuser requiring only :view-data :unrestricted sees tables with that permission"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}
+ :model/Table {table-blocked :id} {:db_id db-id :name "Table3"}
+ :model/PermissionsGroup pg {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg)
+ (t2/delete! :model/DataPermissions :db_id db-id)
+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)
+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)
+ (data-perms/set-table-permission! pg table-1 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg table-1 :perms/create-queries :query-builder)
+ (data-perms/set-table-permission! pg table-2 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg table-2 :perms/create-queries :query-builder-and-native)
+ (data-perms/set-table-permission! pg table-blocked :perms/view-data :blocked)
+ (data-perms/set-table-permission! pg table-blocked :perms/create-queries :no)
+ (is (= #{table-1 table-2}
+ (fetch-visible-table-ids db-id
+ (regular-user-info (mt/user->id :rasta))
+ {:perms/view-data :unrestricted
+ :perms/create-queries :no}
+ :id)))))))
+
+(deftest visible-filter-clause-legacy-no-self-service-test
+ (testing "Non-superuser requiring :view-data :legacy-no-self-service sees tables at that level or above"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}
+ :model/Table {table-blocked :id} {:db_id db-id :name "Table3"}
+ :model/Table {table-legacy :id} {:db_id db-id :name "Table4"}
+ :model/PermissionsGroup pg {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg)
+ (t2/delete! :model/DataPermissions :db_id db-id)
+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)
+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)
+ (data-perms/set-table-permission! pg table-1 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg table-1 :perms/create-queries :query-builder)
+ (data-perms/set-table-permission! pg table-2 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg table-2 :perms/create-queries :query-builder-and-native)
+ (data-perms/set-table-permission! pg table-blocked :perms/view-data :blocked)
+ (data-perms/set-table-permission! pg table-blocked :perms/create-queries :no)
+ (data-perms/set-table-permission! pg table-legacy :perms/view-data :legacy-no-self-service)
+ (data-perms/set-table-permission! pg table-legacy :perms/create-queries :no)
+ (is (= #{table-1 table-2 table-legacy}
+ (fetch-visible-table-ids db-id
+ (regular-user-info (mt/user->id :rasta))
+ {:perms/view-data :legacy-no-self-service
+ :perms/create-queries :no}
+ :id)))))))
+
+(deftest visible-filter-clause-blocked-level-sees-all-test
+ (testing "Non-superuser requiring :view-data :blocked sees all tables since all values are more permissive"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}
+ :model/Table {table-blocked :id} {:db_id db-id :name "Table3"}
+ :model/Table {table-legacy :id} {:db_id db-id :name "Table4"}
+ :model/PermissionsGroup pg {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg)
+ (t2/delete! :model/DataPermissions :db_id db-id)
+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)
+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)
+ (data-perms/set-table-permission! pg table-1 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg table-1 :perms/create-queries :query-builder)
+ (data-perms/set-table-permission! pg table-2 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg table-2 :perms/create-queries :query-builder-and-native)
+ (data-perms/set-table-permission! pg table-blocked :perms/view-data :blocked)
+ (data-perms/set-table-permission! pg table-blocked :perms/create-queries :no)
+ (data-perms/set-table-permission! pg table-legacy :perms/view-data :legacy-no-self-service)
+ (data-perms/set-table-permission! pg table-legacy :perms/create-queries :no)
+ (is (= #{table-1 table-2 table-blocked table-legacy}
+ (fetch-visible-table-ids db-id
+ (regular-user-info (mt/user->id :rasta))
+ {:perms/view-data :blocked
+ :perms/create-queries :no}
+ :id)))))))
+
+(deftest visible-filter-clause-blocked-takes-precedence-test
+ (testing "Blocked permission takes precedence over legacy-no-self-service across groups"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}
+ :model/Table {table-legacy :id} {:db_id db-id :name "Table4"}
+ :model/PermissionsGroup pg1 {}
+ :model/PermissionsGroup pg2 {}]
(perms/add-user-to-group! (mt/user->id :rasta) pg1)
(perms/add-user-to-group! (mt/user->id :rasta) pg2)
-
(t2/delete! :model/DataPermissions :db_id db-id)
-
+ ;; pg1 has legacy-no-self-service for table-legacy
(data-perms/set-database-permission! pg1 db-id :perms/view-data :blocked)
(data-perms/set-database-permission! pg1 db-id :perms/create-queries :no)
-
+ (data-perms/set-table-permission! pg1 table-1 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg1 table-1 :perms/create-queries :query-builder)
+ (data-perms/set-table-permission! pg1 table-2 :perms/view-data :unrestricted)
+ (data-perms/set-table-permission! pg1 table-2 :perms/create-queries :query-builder-and-native)
+ (data-perms/set-table-permission! pg1 table-legacy :perms/view-data :legacy-no-self-service)
+ (data-perms/set-table-permission! pg1 table-legacy :perms/create-queries :no)
+ ;; pg2 blocks table-legacy, which should take precedence
(data-perms/set-database-permission! pg2 db-id :perms/view-data :legacy-no-self-service)
(data-perms/set-database-permission! pg2 db-id :perms/create-queries :no)
-
- (data-perms/set-table-permission! pg1 table-view-data-unrestricted-create-queries-query-builder-id :perms/view-data :unrestricted)
- (data-perms/set-table-permission! pg1 table-view-data-unrestricted-create-queries-query-builder-id :perms/create-queries :query-builder)
-
- (data-perms/set-table-permission! pg1 table-view-data-unrestricted-create-queries-query-builder-and-native-id :perms/view-data :unrestricted)
- (data-perms/set-table-permission! pg1 table-view-data-unrestricted-create-queries-query-builder-and-native-id :perms/create-queries :query-builder-and-native)
-
- (data-perms/set-table-permission! pg1 table-view-data-blocked-create-queries-no-id :perms/view-data :blocked)
- (data-perms/set-table-permission! pg1 table-view-data-blocked-create-queries-no-id :perms/create-queries :no)
-
- (data-perms/set-table-permission! pg1 table-view-data-legacy-no-self-service-create-queries-no-id :perms/view-data :legacy-no-self-service)
- (data-perms/set-table-permission! pg1 table-view-data-legacy-no-self-service-create-queries-no-id :perms/create-queries :no)
-
- (let [user-id (mt/user->id :rasta)
- fetch-visible-ids (fn [user-info permission-mapping table-id-field]
- (let [filter-clause (mi/visible-filter-clause :model/Table table-id-field user-info permission-mapping)]
- (t2/select-pks-set [:model/Table]
- {:where
- [:and
- [:= :db_id db-id]
- filter-clause]})))]
-
- (testing "Superuser should see all tables"
- (let [user-info {:user-id user-id ; User ID doesn't matter for superuser
- :is-superuser? true}
- permission-map {:perms/view-data :unrestricted ; Levels don't matter for superuser
- :perms/create-queries :query-builder}]
- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id
- table-view-data-unrestricted-create-queries-query-builder-and-native-id
- table-view-data-blocked-create-queries-no-id
- table-view-data-legacy-no-self-service-create-queries-no-id}
- (fetch-visible-ids user-info permission-map :id)))))
-
- (testing "Non-superuser should only see tables where they have both view and query permissions (mimics mi/can-read?)"
- (let [user-info {:user-id user-id
- :is-superuser? false}
- permission-map {:perms/view-data :unrestricted
- :perms/create-queries :query-builder}]
- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id
- table-view-data-unrestricted-create-queries-query-builder-and-native-id}
- (fetch-visible-ids user-info permission-map :id))
- "Clause should filter out tables without sufficient view/query permissions")
-
- (testing "using a sequence of fields for the table ID"
- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id
- table-view-data-unrestricted-create-queries-query-builder-and-native-id}
- (fetch-visible-ids user-info permission-map [:coalesce :id :metabase_table.id]))
- "Clause should work with coalesce when a sequence of fields is provided"))
-
- (testing "using a qualified keyword"
- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id
- table-view-data-unrestricted-create-queries-query-builder-and-native-id}
- (fetch-visible-ids user-info permission-map :metabase_table.id))
- "Clause should work with qualified keyword"))))
-
- (testing "Non-superuser requiring only :view-data :unrestricted should see tables 1 and 2"
- (let [user-info {:user-id user-id
- :is-superuser? false}
- permission-map {:perms/view-data :unrestricted
- :perms/create-queries :no}]
- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id
- table-view-data-unrestricted-create-queries-query-builder-and-native-id}
- (fetch-visible-ids user-info permission-map :id))
- "Clause should respect the provided permission levels")))
-
- (testing "Non-superuser requiring only :view-data :legacy-no-self-service should see tables 1, 2, and 4"
- (let [user-info {:user-id user-id
- :is-superuser? false}
- permission-map {:perms/view-data :legacy-no-self-service
- :perms/create-queries :no}]
- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id
- table-view-data-unrestricted-create-queries-query-builder-and-native-id
- table-view-data-legacy-no-self-service-create-queries-no-id}
- (fetch-visible-ids user-info permission-map :id))
- "Clause should respect the provided permission levels")))
-
- (testing "Non-superuser requiring :view-data :blocked should see all tables since all values are more permissive "
- (let [user-info {:user-id user-id
- :is-superuser? false}
- permission-map {:perms/view-data :blocked
- :perms/create-queries :no}]
- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id
- table-view-data-unrestricted-create-queries-query-builder-and-native-id
- table-view-data-blocked-create-queries-no-id
- table-view-data-legacy-no-self-service-create-queries-no-id}
- (fetch-visible-ids user-info permission-map :id))
- "Clause should filter correctly when requiring :blocked level")))
-
- (testing "Blocked table takes precedence over legacy-no-self-service when in any group a user is a member of"
- (data-perms/set-table-permission! pg2 table-view-data-legacy-no-self-service-create-queries-no-id :perms/view-data :blocked)
- (data-perms/set-table-permission! pg2 table-view-data-legacy-no-self-service-create-queries-no-id :perms/create-queries :no)
-
- (let [user-info {:user-id user-id
- :is-superuser? false}
- permission-map {:perms/view-data :legacy-no-self-service
- :perms/create-queries :no}]
- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id
- table-view-data-unrestricted-create-queries-query-builder-and-native-id}
- (fetch-visible-ids user-info permission-map :id))
- "Clause should respect the provided permission levels"))))))))
-
-(deftest visible-filter-clause-db-level-test
- (testing "visible-tables-filter-clause generates a HoneySQL clause that filters tables based on user permissions when only db level perms are set"
- (mt/with-temp [:model/Database {db-id :id} {}
- :model/Table {table-id-1 :id} {:db_id db-id :name "Table1"}
- :model/Table {table-id-2 :id} {:db_id db-id :name "Table2"}
- :model/Table {table-id-3 :id} {:db_id db-id :name "Table3"}]
-
+ (data-perms/set-table-permission! pg2 table-legacy :perms/view-data :blocked)
+ (data-perms/set-table-permission! pg2 table-legacy :perms/create-queries :no)
+ (is (= #{table-1 table-2}
+ (fetch-visible-table-ids db-id
+ (regular-user-info (mt/user->id :rasta))
+ {:perms/view-data :legacy-no-self-service
+ :perms/create-queries :no}
+ :id)))))))
+
+;;; ---------------------------------------- DB-level permissions tests ----------------------------------------
+
+(deftest visible-filter-clause-db-level-superuser-test
+ (testing "Superuser sees all tables with DB-level permissions"
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}
+ :model/Table {table-3 :id} {:db_id db-id :name "Table3"}]
(data-perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)
(data-perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries :no)
-
- (let [user-id (mt/user->id :rasta)
- fetch-visible-ids (fn [user-cfg permission-map table-id-field]
- (let [filter-clause (mi/visible-filter-clause :model/Table table-id-field user-cfg permission-map)]
- (t2/select-pks-set [:model/Table]
- {:where
- [:and
- [:= :db_id db-id]
- filter-clause]})))]
-
- (testing "Superuser should see all tables"
- (let [superuser-cfg {:user-id user-id ; User ID doesn't matter for superuser
- :is-superuser? true}
- permission-map {:perms/view-data :unrestricted ; Levels don't matter for superuser
- :perms/create-queries :query-builder}]
- (is (= #{table-id-1 table-id-2 table-id-3}
- (fetch-visible-ids superuser-cfg permission-map :id)))))
-
- (testing "Non-superuser should only see tables where they have both view and query permissions (mimics mi/can-read?)"
- (let [user-info {:user-id user-id
- :is-superuser? false}
- permission-map {:perms/view-data :unrestricted
- :perms/create-queries :query-builder}]
- (is (empty? ; Expecting empty set if no tables are visible with these perms
- (fetch-visible-ids user-info permission-map :id))
- "Clause should filter out tables without sufficient view/query permissions")
-
- (testing "using an expression for the table ID"
- (is (empty?
- (fetch-visible-ids user-info permission-map [:coalesce :id :metabase_table.id]))
- "Clause should work with coalesce when a sequence of fields is provided"))
-
- (testing "using a qualified keyword"
- (is (empty?
- (fetch-visible-ids user-info permission-map :metabase_table.id))
- "Clause should work with qualified keyword"))))
-
- (testing "Non-superuser requiring only :view-data :unrestricted should see tables 1, 2 and 3"
- (let [user-info {:user-id user-id
- :is-superuser? false}
- permission-map {:perms/view-data :unrestricted
- :perms/create-queries :no}]
- (is (= #{table-id-1 table-id-2 table-id-3}
- (fetch-visible-ids user-info permission-map :id))
- "Clause should respect the provided permission levels")))
-
- (testing "Non-superuser requiring :view-data :blocked should all tables since all values are more permissive "
- (let [user-info {:user-id user-id
- :is-superuser? false}
- permission-map {:perms/view-data :blocked
- :perms/create-queries :no}]
- (is (= #{table-id-1 table-id-2 table-id-3}
- (fetch-visible-ids user-info permission-map :id))
- "Clause should filter correctly when requiring :blocked level")))))))
+ (is (= #{table-1 table-2 table-3}
+ (fetch-visible-table-ids db-id
+ (superuser-info (mt/user->id :rasta))
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder}
+ :id))))))
+
+(deftest visible-filter-clause-db-level-no-query-perms-test
+ (testing "Non-superuser with DB-level view but no query perms sees no tables when both are required"
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {_table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {_table-2 :id} {:db_id db-id :name "Table2"}
+ :model/Table {_table-3 :id} {:db_id db-id :name "Table3"}]
+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries :no)
+ (is (empty? (fetch-visible-table-ids db-id
+ (regular-user-info (mt/user->id :rasta))
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder}
+ :id))))))
+
+(deftest visible-filter-clause-db-level-coalesce-test
+ (testing "Filter clause with coalesce works for DB-level permissions"
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {_table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {_table-2 :id} {:db_id db-id :name "Table2"}]
+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries :no)
+ (is (empty? (fetch-visible-table-ids db-id
+ (regular-user-info (mt/user->id :rasta))
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder}
+ [:coalesce :id :metabase_table.id]))))))
+
+(deftest visible-filter-clause-db-level-qualified-keyword-test
+ (testing "Filter clause with qualified keyword works for DB-level permissions"
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {_table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {_table-2 :id} {:db_id db-id :name "Table2"}]
+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries :no)
+ (is (empty? (fetch-visible-table-ids db-id
+ (regular-user-info (mt/user->id :rasta))
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder}
+ :metabase_table.id))))))
+
+(deftest visible-filter-clause-db-level-view-only-test
+ (testing "Non-superuser requiring only :view-data :unrestricted sees all tables with DB-level unrestricted"
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}
+ :model/Table {table-3 :id} {:db_id db-id :name "Table3"}]
+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries :no)
+ (is (= #{table-1 table-2 table-3}
+ (fetch-visible-table-ids db-id
+ (regular-user-info (mt/user->id :rasta))
+ {:perms/view-data :unrestricted
+ :perms/create-queries :no}
+ :id))))))
+
+(deftest visible-filter-clause-db-level-blocked-test
+ (testing "Non-superuser requiring :blocked sees all tables since all values are more permissive"
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}
+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}
+ :model/Table {table-3 :id} {:db_id db-id :name "Table3"}]
+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries :no)
+ (is (= #{table-1 table-2 table-3}
+ (fetch-visible-table-ids db-id
+ (regular-user-info (mt/user->id :rasta))
+ {:perms/view-data :blocked
+ :perms/create-queries :no}
+ :id))))))
(deftest prevent-metabase-transform-data-source-change-test
(testing "Cannot change data_source from metabase-transform"
diff --git a/test/metabase/warehouses/models/database_test.clj b/test/metabase/warehouses/models/database_test.clj
index e6aba937912..44034658745 100644
--- a/test/metabase/warehouses/models/database_test.clj
+++ b/test/metabase/warehouses/models/database_test.clj
@@ -620,8 +620,24 @@
:tables
(#(map :name %))))))
-(deftest visible-filter-clause-test
- (testing "Database visible-filter-clause generates a HoneySQL clause that filters databases based on user permissions"
+;;; ---------------------------------------- visible-filter-clause tests ----------------------------------------
+
+(defn- fetch-visible-db-ids
+ "Helper to fetch visible database IDs using visible-filter-clause.
+ Handles the :with/:clause return value format."
+ [db-ids user-info permission-mapping column-field]
+ (let [{:keys [clause]} (mi/visible-filter-clause :model/Database column-field user-info permission-mapping)]
+ (t2/select-pks-set :model/Database
+ {:where [:and
+ clause
+ [:in :id db-ids]]})))
+
+(def ^:private default-permission-mapping
+ {:perms/view-data :unrestricted
+ :perms/create-queries :query-builder})
+
+(deftest visible-filter-clause-superuser-test
+ (testing "Superuser should see all databases"
(mt/with-no-data-perms-for-all-users!
(mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}
:model/Database {db2-id :id} {:name "Database 2"}
@@ -631,130 +647,204 @@
:model/Table _ {:db_id db3-id :name "Table3"}
:model/PermissionsGroup pg1 {}
:model/PermissionsGroup pg2 {}]
-
- ;; Set up permissions: user has access to db1 and db2 via different groups, but not db3
(perms/add-user-to-group! (mt/user->id :rasta) pg1)
(perms/add-user-to-group! (mt/user->id :rasta) pg2)
-
- ;; Clear existing permissions for our test databases only
(t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])
-
- ;; Grant permissions to specific databases
(data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)
(data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)
-
(data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)
(data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)
-
- ;; No permissions for db3
(data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)
(data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)
-
- (let [user-id (mt/user->id :rasta)
- fetch-visible-ids (fn [user-info permission-mapping column-field]
- (let [filter-clause (mi/visible-filter-clause :model/Database column-field user-info permission-mapping)]
- (t2/select-pks-set :model/Database
- {:where [:and
- filter-clause
- [:in :id [db1-id db2-id db3-id]]]})))] ; Only check our test databases
-
- (testing "Superuser should see all databases"
- (let [user-info {:user-id user-id :is-superuser? true}
- permission-mapping {:perms/view-data :unrestricted
- :perms/create-queries :query-builder}]
- (is (= #{db1-id db2-id db3-id}
- (fetch-visible-ids user-info permission-mapping :id))
- "Superuser should see all databases")))
-
- (testing "Non-superuser should only see databases they have permissions for"
- (let [user-info {:user-id user-id :is-superuser? false}
- permission-mapping {:perms/view-data :unrestricted
- :perms/create-queries :query-builder}]
- (is (= #{db1-id db2-id}
- (fetch-visible-ids user-info permission-mapping :id))
- "Non-superuser should only see databases with sufficient permissions")))
-
- (testing "Using qualified column name"
- (let [user-info {:user-id user-id :is-superuser? false}
- permission-mapping {:perms/view-data :unrestricted
- :perms/create-queries :query-builder}]
- (is (= #{db1-id db2-id}
- (fetch-visible-ids user-info permission-mapping :metabase_database.id))
- "Should work with qualified column names")))
-
- (testing "Using column expression"
- (let [user-info {:user-id user-id :is-superuser? false}
- permission-mapping {:perms/view-data :unrestricted
- :perms/create-queries :query-builder}]
- (is (= #{db1-id db2-id}
- (fetch-visible-ids user-info permission-mapping [:coalesce :id :metabase_database.id]))
- "Should work with column expressions")))
-
- (testing "Different permission levels"
- (testing "Requiring only view-data permissions"
- (let [user-info {:user-id user-id :is-superuser? false}
- permission-mapping {:perms/view-data :unrestricted
- :perms/create-queries :no}]
- (is (= #{db1-id db2-id}
- (fetch-visible-ids user-info permission-mapping :id))
- "Should include databases where user has view permissions")))
-
- (testing "Requiring blocked level permissions (most permissive)"
- (let [user-info {:user-id user-id :is-superuser? false}
- permission-mapping {:perms/view-data :blocked
- :perms/create-queries :no}]
- (is (= #{db1-id db2-id db3-id}
- (fetch-visible-ids user-info permission-mapping :id))
- "Should include all databases when requiring only blocked level"))))
-
- (testing "User with no permissions"
- ;; Remove user from groups we added (avoid touching All Users group)
- (t2/delete! :model/PermissionsGroupMembership
- :user_id user-id
- :group_id [:in [(:id pg1) (:id pg2)]])
-
- (let [user-info {:user-id user-id :is-superuser? false}
- permission-mapping {:perms/view-data :unrestricted
- :perms/create-queries :query-builder}]
- (is (empty? (fetch-visible-ids user-info permission-mapping :id))
- "User with no group memberships should see no databases"))))))))
+ (is (= #{db1-id db2-id db3-id}
+ (fetch-visible-db-ids [db1-id db2-id db3-id]
+ {:user-id (mt/user->id :rasta) :is-superuser? true}
+ default-permission-mapping
+ :id)))))))
+
+(deftest visible-filter-clause-non-superuser-test
+ (testing "Non-superuser should only see databases they have permissions for"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}
+ :model/Database {db2-id :id} {:name "Database 2"}
+ :model/Database {db3-id :id} {:name "Database 3"}
+ :model/Table _ {:db_id db1-id :name "Table1"}
+ :model/Table _ {:db_id db2-id :name "Table2"}
+ :model/Table _ {:db_id db3-id :name "Table3"}
+ :model/PermissionsGroup pg1 {}
+ :model/PermissionsGroup pg2 {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg1)
+ (perms/add-user-to-group! (mt/user->id :rasta) pg2)
+ (t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])
+ (data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)
+ (data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)
+ (is (= #{db1-id db2-id}
+ (fetch-visible-db-ids [db1-id db2-id db3-id]
+ {:user-id (mt/user->id :rasta) :is-superuser? false}
+ default-permission-mapping
+ :id)))))))
+
+(deftest visible-filter-clause-qualified-column-test
+ (testing "Should work with qualified column names"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}
+ :model/Database {db2-id :id} {:name "Database 2"}
+ :model/Database {db3-id :id} {:name "Database 3"}
+ :model/Table _ {:db_id db1-id :name "Table1"}
+ :model/Table _ {:db_id db2-id :name "Table2"}
+ :model/Table _ {:db_id db3-id :name "Table3"}
+ :model/PermissionsGroup pg1 {}
+ :model/PermissionsGroup pg2 {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg1)
+ (perms/add-user-to-group! (mt/user->id :rasta) pg2)
+ (t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])
+ (data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)
+ (data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)
+ (is (= #{db1-id db2-id}
+ (fetch-visible-db-ids [db1-id db2-id db3-id]
+ {:user-id (mt/user->id :rasta) :is-superuser? false}
+ default-permission-mapping
+ :metabase_database.id)))))))
+
+(deftest visible-filter-clause-column-expression-test
+ (testing "Should work with column expressions"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}
+ :model/Database {db2-id :id} {:name "Database 2"}
+ :model/Database {db3-id :id} {:name "Database 3"}
+ :model/Table _ {:db_id db1-id :name "Table1"}
+ :model/Table _ {:db_id db2-id :name "Table2"}
+ :model/Table _ {:db_id db3-id :name "Table3"}
+ :model/PermissionsGroup pg1 {}
+ :model/PermissionsGroup pg2 {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg1)
+ (perms/add-user-to-group! (mt/user->id :rasta) pg2)
+ (t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])
+ (data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)
+ (data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)
+ (is (= #{db1-id db2-id}
+ (fetch-visible-db-ids [db1-id db2-id db3-id]
+ {:user-id (mt/user->id :rasta) :is-superuser? false}
+ default-permission-mapping
+ [:coalesce :id :metabase_database.id])))))))
+
+(deftest visible-filter-clause-view-data-only-test
+ (testing "Requiring only view-data permissions should include databases where user has view permissions"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}
+ :model/Database {db2-id :id} {:name "Database 2"}
+ :model/Database {db3-id :id} {:name "Database 3"}
+ :model/Table _ {:db_id db1-id :name "Table1"}
+ :model/Table _ {:db_id db2-id :name "Table2"}
+ :model/Table _ {:db_id db3-id :name "Table3"}
+ :model/PermissionsGroup pg1 {}
+ :model/PermissionsGroup pg2 {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg1)
+ (perms/add-user-to-group! (mt/user->id :rasta) pg2)
+ (t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])
+ (data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)
+ (data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)
+ (is (= #{db1-id db2-id}
+ (fetch-visible-db-ids [db1-id db2-id db3-id]
+ {:user-id (mt/user->id :rasta) :is-superuser? false}
+ {:perms/view-data :unrestricted
+ :perms/create-queries :no}
+ :id)))))))
+
+(deftest visible-filter-clause-blocked-level-test
+ (testing "Requiring blocked level permissions (most permissive) should include all databases"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}
+ :model/Database {db2-id :id} {:name "Database 2"}
+ :model/Database {db3-id :id} {:name "Database 3"}
+ :model/Table _ {:db_id db1-id :name "Table1"}
+ :model/Table _ {:db_id db2-id :name "Table2"}
+ :model/Table _ {:db_id db3-id :name "Table3"}
+ :model/PermissionsGroup pg1 {}
+ :model/PermissionsGroup pg2 {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg1)
+ (perms/add-user-to-group! (mt/user->id :rasta) pg2)
+ (t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])
+ (data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)
+ (data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)
+ (is (= #{db1-id db2-id db3-id}
+ (fetch-visible-db-ids [db1-id db2-id db3-id]
+ {:user-id (mt/user->id :rasta) :is-superuser? false}
+ {:perms/view-data :blocked
+ :perms/create-queries :no}
+ :id)))))))
+
+(deftest visible-filter-clause-no-permissions-test
+ (testing "User with no group memberships should see no databases"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}
+ :model/Database {db2-id :id} {:name "Database 2"}
+ :model/Database {db3-id :id} {:name "Database 3"}
+ :model/Table _ {:db_id db1-id :name "Table1"}
+ :model/Table _ {:db_id db2-id :name "Table2"}
+ :model/Table _ {:db_id db3-id :name "Table3"}
+ :model/PermissionsGroup pg1 {}
+ :model/PermissionsGroup pg2 {}]
+ (perms/add-user-to-group! (mt/user->id :rasta) pg1)
+ (perms/add-user-to-group! (mt/user->id :rasta) pg2)
+ (t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])
+ (data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)
+ (data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)
+ (data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)
+ ;; Remove user from groups we added (avoid touching All Users group)
+ (t2/delete! :model/PermissionsGroupMembership
+ :user_id (mt/user->id :rasta)
+ :group_id [:in [(:id pg1) (:id pg2)]])
+ (is (empty? (fetch-visible-db-ids [db1-id db2-id db3-id]
+ {:user-id (mt/user->id :rasta) :is-superuser? false}
+ default-permission-mapping
+ :id)))))))
(deftest visible-filter-clause-table-level-permissions-test
- (testing "Database visible-filter-clause with table-level permissions"
+ (testing "Database should be visible when user has access to at least one table"
(mt/with-no-data-perms-for-all-users!
(mt/with-temp [:model/Database {db-id :id} {:name "Test Database"}
:model/Table {table1-id :id} {:db_id db-id :name "Table1"}
:model/Table {table2-id :id} {:db_id db-id :name "Table2"}
:model/Table _ {:db_id db-id :name "Table3"}
:model/PermissionsGroup pg {}]
-
(perms/add-user-to-group! (mt/user->id :rasta) pg)
-
;; Clear existing permissions for our test database only
(t2/delete! :model/DataPermissions :db_id db-id)
-
;; Block database-level access
(data-perms/set-database-permission! pg db-id :perms/view-data :blocked)
(data-perms/set-database-permission! pg db-id :perms/create-queries :no)
-
- ;; Grant table-level permissions to only table1 and table2
+ ;; Grant table-level permissions to only table1 and table2 (table3 remains blocked)
(data-perms/set-table-permission! pg table1-id :perms/view-data :unrestricted)
(data-perms/set-table-permission! pg table1-id :perms/create-queries :query-builder)
-
(data-perms/set-table-permission! pg table2-id :perms/view-data :unrestricted)
(data-perms/set-table-permission! pg table2-id :perms/create-queries :query-builder)
- ;; table3 remains blocked
-
- (let [user-id (mt/user->id :rasta)
- user-info {:user-id user-id :is-superuser? false}
- permission-mapping {:perms/view-data :unrestricted
- :perms/create-queries :query-builder}
- filter-clause (mi/visible-filter-clause :model/Database :id user-info permission-mapping)
- visible-db-ids (t2/select-pks-set :model/Database
- {:where [:and
- filter-clause
- [:= :id db-id]]})] ; Only check our test database
-
- (is (contains? visible-db-ids db-id)
- "Database should be visible when user has access to at least one table"))))))
+ (is (contains? (fetch-visible-db-ids [db-id]
+ {:user-id (mt/user->id :rasta) :is-superuser? false}
+ default-permission-mapping
+ :id)
+ db-id))))))