The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fix: improve search performance with tables (#67055)
Edward Pagetd2212003ef3c
diff --git a/dev/src/dev/search_perf.clj b/dev/src/dev/search_perf.cljnew file mode 100644index 00000000000..2d3144940d4--- /dev/null+++ b/dev/src/dev/search_perf.clj@@ -0,0 +1,511 @@+(ns dev.search-perf+ "Performance testing utilities for search with large numbers of tables,+ users, and permission groups.++ Example usage:+ ;; Run full benchmark with default settings (10000 tables, 10 groups, 100 users)+ (run-full-benchmark!)++ ;; Run with custom settings+ (run-full-benchmark! {:num-tables 10000+ :num-groups 10+ :num-users 100+ :iterations 5})++ ;; Manual setup for interactive testing+ (def test-env (create-test-environment! {:num-tables 1000+ :num-groups 5+ :num-users 50}))+ (timed-search! \"table\" :user-id (:user-id test-env))+ (cleanup-test-environment! test-env)"+ (:require+ [clojure.string :as str]+ [dev.add-load :as add-load]+ [metabase.api.common :as api]+ [metabase.permissions.core :as perms]+ [metabase.permissions.util :as perms-util]+ [metabase.request.core :as request]+ [metabase.search.config :as search.config]+ [metabase.search.engine :as search.engine]+ [metabase.search.impl :as search.impl]+ [metabase.search.ingestion :as search.ingestion]+ [metabase.util :as u]+ [metabase.util.random :as u.random]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++;;; -------------------------------------------- Setup Utilities: Database & Tables ------------------------------------++(defn- generate-table-name+ "Generate a table name with a prefix and index for identification."+ [prefix idx]+ (format "%s_table_%05d" prefix idx))++(defn create-test-database!+ "Create a test database for performance testing.+ Returns the database ID."+ []+ (let [{:keys [db-id]} (add-load/from-script+ [[:model/Database {:?/db-id :id}+ {:name (str "Search Perf Test DB " (u.random/random-name))+ :engine :h2+ :details {}}]])]+ (perms/set-database-permission! (perms/all-users-group) db-id :perms/view-data :blocked)+ (perms/set-database-permission! (perms/all-users-group) db-id :perms/create-queries :no)+ db-id))++(defn create-tables-batch!+ "Create a batch of tables for the given database.+ Returns the number of tables created."+ [db-id prefix start-idx cnt]+ (let [script (vec+ (for [i (range start-idx (+ start-idx cnt))]+ [:model/Table {}+ {:db_id db-id+ :name (generate-table-name prefix i)+ :description (format "Performance test table %d for search benchmarking" i)+ :active true}]))]+ (add-load/from-script script)+ cnt))++(defn create-test-database-with-tables!+ "Create a test database with the specified number of tables.+ Tables are created in batches for better performance.++ Options:+ :batch-size - Number of tables per batch (default 500)+ :prefix - Prefix for table names (default \"perf_test\")++ Returns a map with :db-id and :table-ids."+ ([num-tables]+ (create-test-database-with-tables! num-tables {}))+ ([num-tables {:keys [batch-size prefix]+ :or {batch-size 500+ prefix "perf_test"}}]+ (let [db-id (create-test-database!)+ num-batches (Math/ceil (/ num-tables batch-size))]+ (println (format "Creating %d tables in %d batches of %d..."+ num-tables (int num-batches) batch-size))+ (let [timer (u/start-timer)]+ (doseq [batch-num (range (int num-batches))]+ (let [start-idx (* batch-num batch-size)+ batch-count (min batch-size (- num-tables start-idx))]+ (create-tables-batch! db-id prefix start-idx batch-count)+ (when (zero? (mod (inc batch-num) 10))+ (println (format " Created %d/%d tables..." (+ start-idx batch-count) num-tables)))))+ (let [elapsed-ms (u/since-ms timer)+ ;; Get all table IDs for this database+ table-ids (t2/select-pks-vec :model/Table :db_id db-id)]+ (println (format "Created %d tables in %.2f seconds (%.1f tables/sec)"+ num-tables+ (/ elapsed-ms 1000.0)+ (/ (* num-tables 1000.0) elapsed-ms)))+ {:db-id db-id+ :table-ids table-ids})))))++;;; -------------------------------------------- Setup Utilities: Groups & Users ---------------------------------------++(defn create-permission-groups!+ "Create n permission groups for testing.+ Returns a vector of group IDs."+ [n run-id]+ (println (format "Creating %d permission groups..." n))+ (let [timer (u/start-timer)+ prefix (str "Search Perf Group " run-id " ")+ script (vec+ (for [i (range n)]+ [:model/PermissionsGroup {}+ {:name (str prefix i)}]))+ _ (add-load/from-script script)+ ;; Query for groups with our name pattern+ group-ids (t2/select-pks-vec :model/PermissionsGroup :name [:like (str prefix "%")])+ elapsed-ms (u/since-ms timer)]+ (println (format "Created %d groups in %.2f seconds" n (/ elapsed-ms 1000.0)))+ group-ids))++(defn create-users-with-memberships!+ "Create n users and assign them to permission groups.+ Each user is assigned to a subset of groups based on their index.+ Users are also added to the All Users group automatically by the model.++ Returns a vector of user IDs."+ [n group-ids run-id]+ (println (format "Creating %d users with group memberships..." n))+ (let [timer (u/start-timer)+ num-groups (count group-ids)+ email-domain (str "perftest" run-id ".example.com")+ ;; Create users first+ user-script (vec+ (for [i (range n)]+ [:model/User {}+ {:first_name (format "PerfUser%d" i)+ :last_name "Test"+ :email (format "user%d@%s" i email-domain)+ :password "password123"}]))+ _ (add-load/from-script user-script)+ ;; Query for users by email pattern+ user-ids (t2/select-pks-vec :model/User :email [:like (str "%@" email-domain)])]++ ;; Create group memberships - assign each user to some groups+ ;; User i gets assigned to groups where (mod i num-groups) matches certain patterns+ (println " Assigning users to groups...")+ (let [membership-script+ (vec+ (for [user-idx (range n)+ group-idx (range num-groups)+ ;; Assign user to ~30% of groups based on a deterministic pattern+ :when (zero? (mod (+ user-idx group-idx) 3))]+ [:model/PermissionsGroupMembership {}+ {:user_id (nth user-ids user-idx)+ :group_id (nth group-ids group-idx)}]))]+ (when (seq membership-script)+ (add-load/from-script membership-script)))++ (let [elapsed-ms (u/since-ms timer)]+ (println (format "Created %d users with memberships in %.2f seconds" n (/ elapsed-ms 1000.0))))+ user-ids))++(defn grant-table-permissions-to-groups!+ "Grant table-level permissions to groups.+ Each group gets permissions to a subset of tables based on deterministic assignment.+ This creates a realistic permission scenario where different groups have access+ to different tables.++ Assignment strategy:+ - Each group gets access to ~5% of tables+ - Tables are assigned based on (mod table-idx 20) pattern with group offset+ - This ensures overlapping but distinct permission sets"+ [table-ids group-ids]+ (println (format "Granting table-level permissions to %d groups for %d tables..."+ (count group-ids) (count table-ids)))+ (let [timer (u/start-timer)+ num-groups (count group-ids)+ ;; Build permission assignments+ ;; Each group gets ~5% of tables (1 in 20) with offset based on group index+ assignments (for [group-idx (range num-groups)+ :let [group-id (nth group-ids group-idx)+ ;; Each group gets ~5% of tables with some overlap+ ;; Using mod 20 gives 5%, offset by group-idx for variation+ assigned-tables (filterv+ (fn [table-idx]+ ;; Deterministic pattern: group gets every 20th table+ ;; offset by group index to create variation between groups+ (zero? (mod (+ table-idx group-idx) 20)))+ (range (count table-ids)))]]+ {:group-id group-id+ :group-idx group-idx+ :table-indices assigned-tables+ :num-tables (count assigned-tables)})]++ ;; Grant permissions - batch by group for efficiency+ (doseq [{:keys [group-id table-indices]} assignments]+ (let [table-perms-view (into {} (for [tidx table-indices]+ [(nth table-ids tidx) :unrestricted]))+ table-perms-query (into {} (for [tidx table-indices]+ [(nth table-ids tidx) :query-builder]))]+ ;; Use batch permission setting+ (when (seq table-perms-view)+ (perms/set-table-permissions! group-id :perms/view-data table-perms-view)+ (perms/set-table-permissions! group-id :perms/create-queries table-perms-query))))++ (let [elapsed-ms (u/since-ms timer)]+ (println (format "Granted table permissions in %.2f seconds" (/ elapsed-ms 1000.0))))))++;;; -------------------------------------------- Test Environment Management -------------------------------------------++(defn create-test-environment!+ "Create a complete test environment with database, tables, groups, users, and permissions.++ Options:+ :num-tables - Number of tables to create (default 10000)+ :num-groups - Number of permission groups (default 10)+ :num-users - Number of users (default 100)+ :prefix - Table name prefix (default \"perf_test\")++ Returns a map containing:+ :db-id - Database ID+ :table-ids - Vector of table IDs+ :group-ids - Vector of group IDs+ :user-ids - Vector of user IDs+ :user-id - ID of a sample non-admin user for testing+ :run-id - Unique identifier for this test run"+ ([]+ (create-test-environment! {}))+ ([{:keys [num-tables num-groups num-users prefix]+ :or {num-tables 10000+ num-groups 10+ num-users 100+ prefix "perf_test"}}]+ (println "\n=== Creating Test Environment ===")+ (let [timer (u/start-timer)+ run-id (u.random/random-name)+ ;; Create database and tables+ {:keys [db-id table-ids]} (create-test-database-with-tables! num-tables {:prefix prefix})+ ;; Create permission groups+ group-ids (create-permission-groups! num-groups run-id)+ ;; Grant table-level permissions to groups+ _ (grant-table-permissions-to-groups! table-ids group-ids)+ ;; Create users with group memberships+ user-ids (create-users-with-memberships! num-users group-ids run-id)+ ;; Pick a user in the middle for testing+ test-user-id (nth user-ids (quot num-users 2))+ elapsed-ms (u/since-ms timer)]+ (println (format "Test environment created in %.2f seconds" (/ elapsed-ms 1000.0)))+ (println (format " Run ID: %s" run-id))+ (println (format " Database: %d" db-id))+ (println (format " Tables: %d" num-tables))+ (println (format " Groups: %d" num-groups))+ (println (format " Users: %d" num-users))+ (println (format " Test user ID: %d" test-user-id))+ {:db-id db-id+ :table-ids table-ids+ :group-ids group-ids+ :user-ids user-ids+ :user-id test-user-id+ :run-id run-id+ :num-tables num-tables})))++(defn cleanup-test-environment!+ "Delete all resources created by create-test-environment!"+ [{:keys [db-id group-ids user-ids]}]+ (println "\n=== Cleaning Up Test Environment ===")+ (let [timer (u/start-timer)]+ ;; Delete group memberships first (foreign key constraints)+ (when (seq user-ids)+ (println (format "Deleting memberships for %d users..." (count user-ids)))+ (t2/delete! :model/PermissionsGroupMembership :user_id [:in user-ids] :group_id [:<> 1]))+ ;; Delete users+ (when (seq user-ids)+ (println (format "Deleting %d users..." (count user-ids)))+ (t2/delete! :model/User :id [:in user-ids]))+ ;; Delete data permissions for groups+ (when (seq group-ids)+ (println (format "Deleting permissions for %d groups..." (count group-ids)))+ (t2/delete! :model/DataPermissions :group_id [:in group-ids]))+ ;; Delete groups+ (when (seq group-ids)+ (println (format "Deleting %d groups..." (count group-ids)))+ (t2/delete! :model/PermissionsGroup :id [:in group-ids]))+ ;; Delete tables and database+ (when db-id+ (let [table-count (t2/count :model/Table :db_id db-id)]+ (println (format "Deleting database %d and %d tables..." db-id table-count))+ (t2/delete! :model/Table :db_id db-id)+ (t2/delete! :model/Database :id db-id)))+ (let [elapsed-ms (u/since-ms timer)]+ (println (format "Cleanup completed in %.2f seconds" (/ elapsed-ms 1000.0))))))++;;; ------------------------------------------------ Search Utilities -----------------------------------------------++(defn- search-context-for-user+ "Build a search context for a specific user."+ [user-id search-string models]+ (request/with-current-user user-id+ (search.impl/search-context+ {:current-user-id api/*current-user-id*+ :current-user-perms @api/*current-user-permissions-set*+ :is-superuser? api/*is-superuser?*+ :is-impersonated-user? (perms-util/impersonated-user?)+ :is-sandboxed-user? (perms-util/impersonated-user?)+ :archived false+ :context :default+ :search-string search-string+ :models models+ :model-ancestors? false})))++(defn search!+ "Execute a search and return the results.++ Options:+ :user-id - User ID to run search as (required)+ :models - Models to search (default all models)"+ [search-string & {:keys [user-id models]+ :or {models search.config/all-models}}]+ (assert user-id "user-id is required for search!")+ (let [ctx (search-context-for-user user-id search-string models)]+ (search.engine/results ctx)))++(defn timed-search!+ "Execute a search and return timing information along with result count.++ Options:+ :user-id - User ID to run search as (required)+ :models - Models to search (default all models)++ Returns {:elapsed-ms <time> :result-count <count>}"+ [search-string & {:keys [user-id models]+ :or {models search.config/all-models}}]+ (assert user-id "user-id is required for timed-search!")+ (let [timer (u/start-timer)+ results (search! search-string :user-id user-id :models models)+ elapsed (u/since-ms timer)]+ {:elapsed-ms elapsed+ :result-count (count results)}))++;;; ------------------------------------------------ Benchmark Functions --------------------------------------------++(defn run-search-benchmark!+ "Run a search benchmark with the given search string.++ Options:+ :user-id - User ID to run search as (required)+ :iterations - Number of search iterations (default 5)+ :models - Models to search (default all models)+ :warmup - Number of warmup iterations (default 2)++ Returns a map with benchmark statistics."+ ([search-string opts]+ (let [{:keys [user-id iterations warmup models]+ :or {iterations 5+ warmup 2+ models search.config/all-models}} opts]+ (assert user-id "user-id is required for benchmark!")+ (println (format "\nRunning search benchmark for: \"%s\" (user-id: %d)" search-string user-id))+ (println (format " Warmup iterations: %d" warmup))+ (println (format " Benchmark iterations: %d" iterations))++ ;; Warmup+ (when (pos? warmup)+ (println " Warming up...")+ (dotimes [_ warmup]+ (search! search-string :user-id user-id :models models)))++ ;; Benchmark+ (println " Running benchmark...")+ (let [results (vec (for [i (range iterations)]+ (let [result (timed-search! search-string :user-id user-id :models models)]+ (println (format " Iteration %d: %.2fms (%d results)"+ (inc i)+ (:elapsed-ms result)+ (:result-count result)))+ result)))+ times (mapv :elapsed-ms results)+ avg (/ (reduce + times) (count times))+ min-t (apply min times)+ max-t (apply max times)+ sorted (sort times)+ p50 (nth sorted (int (* 0.5 (count sorted))))+ p95 (nth sorted (int (* 0.95 (dec (count sorted)))))]+ (println "\n Results:")+ (println (format " Average: %.2fms" avg))+ (println (format " Min: %.2fms" min-t))+ (println (format " Max: %.2fms" max-t))+ (println (format " P50: %.2fms" p50))+ (println (format " P95: %.2fms" p95))+ {:search-string search-string+ :user-id user-id+ :iterations iterations+ :result-count (:result-count (first results))+ :avg-ms avg+ :min-ms min-t+ :max-ms max-t+ :p50-ms p50+ :p95-ms p95+ :all-times-ms times}))))++(defn run-full-benchmark!+ "Run a full benchmark suite with multiple search queries.++ Creates a test environment with database, tables, groups, and users,+ runs benchmarks with various search patterns using a non-admin user,+ and optionally cleans up the test data.++ Options:+ :num-tables - Number of tables to create (default 10000)+ :num-groups - Number of permission groups (default 10)+ :num-users - Number of users (default 100)+ :iterations - Benchmark iterations per query (default 5)+ :cleanup? - Whether to cleanup after (default true)+ :prefix - Table name prefix (default \"perf_test\")++ Returns benchmark results for all queries."+ ([]+ (run-full-benchmark! {}))+ ([{:keys [num-tables num-groups num-users iterations cleanup? prefix]+ :or {num-tables 10000+ num-groups 10+ num-users 100+ iterations 5+ cleanup? true+ prefix "perf_test"}}]+ (println (format "\n%s" (str/join (repeat 60 "="))))+ (println "SEARCH PERFORMANCE BENCHMARK (Non-Admin User)")+ (println (str/join (repeat 60 "=")))+ (println "\nConfiguration:")+ (println (format " Tables: %d" num-tables))+ (println (format " Permission Groups: %d" num-groups))+ (println (format " Users: %d" num-users))+ (println (format " Iterations per query: %d" iterations))++ (let [test-env (create-test-environment! {:num-tables num-tables+ :num-groups num-groups+ :num-users num-users+ :prefix prefix})+ user-id (:user-id test-env)+ ;; Wait for search index to update if using async indexing+ _ (binding [search.ingestion/*force-sync* true]+ (Thread/sleep 1000))+ queries [;; Exact prefix match+ (str prefix "_table_00001")+ ;; Partial prefix match+ (str prefix "_table")+ ;; Generic term that should match descriptions+ "performance test"+ ;; Broad search+ "table"+ ;; Very specific search+ (str prefix "_table_05000")]]+ (try+ (let [results (vec (for [query queries]+ (run-search-benchmark! query {:user-id user-id+ :iterations iterations})))]+ (println (format "\n%s" (str/join (repeat 60 "="))))+ (println "SUMMARY")+ (println (str/join (repeat 60 "=")))+ (doseq [{:keys [search-string avg-ms result-count]} results]+ (println (format " \"%s\": %.2fms avg, %d results"+ search-string avg-ms result-count)))+ {:results results+ :test-env test-env})+ (finally+ (when cleanup?+ (cleanup-test-environment! test-env)))))))++(comment+ ;; Example usage:++ ;; Quick test with fewer tables/users+ (run-full-benchmark! {:num-tables 1+ :num-groups 5+ :num-users 4+ :iterations 1})++ ;; Full benchmark with 10,000 tables+ (run-full-benchmark! {:num-tables 10000+ :num-groups 25+ :num-users 100+ :iterations 10})++ ;; {:p95-ms 539.613833, :result-count 10000, :avg-ms 536.6528708, :min-ms 529.310541, :all-times-ms [548.471834 534.8465 538.846875 536.401583 535.707 ...], ...}+ ;; {:p95-ms 366.195458, :result-count 10000, :avg-ms 360.5523999, :min-ms 352.951208, :all-times-ms [355.839542 366.195458 352.951208 356.9155 362.09225 ...], ...}+ ;; {:p95-ms 110.475333, :result-count 4000, :avg-ms 108.0612666, :min-ms 105.713041, :all-times-ms [112.747333 108.450292 107.548208 105.93625 108.159375 ...], ...}++ ;; Manual setup for interactive testing+ (def test-env (create-test-environment! {:num-tables 1000+ :num-groups 35+ :num-users 200}))++ ;; Run individual searches with the non-admin user+ (e/explain+ (timed-search! "perf_test_table" :user-id (:user-id test-env)))+ (timed-search! "table" :user-id (:user-id test-env) :models #{"table"})++ (require '[dev.explain-analyze :as e])+ ;; Try with different users+ (e/explain+ (timed-search! "table" :user-id (first (:user-ids test-env))))+ (timed-search! "table" :user-id (last (:user-ids test-env)))++ ;; Cleanup+ (cleanup-test-environment! test-env))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.cljindex 4db0d87304b..cf5f04dae83 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj@@ -228,7 +228,7 @@- Collection-based (:model/Card, :model/Dashboard, :model/Document, :model/NativeQuerySnippet):Uses collection/visible-collection-filter-clause for collection filtering and adds archived entity filtering.Native query snippets have additional restrictions for sandboxed users.- - Table: Uses mi/visible-filter-clause with appropriate permissions and filters by active/visibility_type.+ - Table: Uses perms/visible-table-filter-select with appropriate permissions and filters by active/visibility_type.Follows search API conventions: active=true AND visibility_type=nil for non-archived tables.Note: archived_at is not checked separately as archived tables always have active=false."([entity-type-field entity-id-field]@@ -279,13 +279,13 @@[:in entity-id-field {:select [:id]:from [table-name]:where [:and- (mi/visible-filter-clause- model- id-column- {:user-id api/*current-user-id*- :is-superuser? api/*is-superuser?*}- {:perms/view-data :unrestricted- :perms/create-queries :query-builder})+ [:in id-column+ (perms/visible-table-filter-select+ :id+ {:user-id api/*current-user-id*+ :is-superuser? api/*is-superuser?*}+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder})](case include-archived-items:exclude [:and[:= active-column true]@@ -305,13 +305,16 @@[:in table-id-column{:select [:metabase_table.id]:from [:metabase_table]- :where (mi/visible-filter-clause- :model/Table- :metabase_table.id- {:user-id api/*current-user-id*- :is-superuser? api/*is-superuser?*}- {:perms/view-data :unrestricted- :perms/create-queries :query-builder})}]+ ;; using this clause because we had to change the mi/visible-filter-clause+ ;; to allow returning CTE based filters+ ;; TODO(ed 2025-12-16: support using CTES in filters in dependency graph)+ :where [:in :metabase_table.id+ (perms/visible-table-filter-select+ :id+ {:user-id api/*current-user-id*+ :is-superuser? api/*is-superuser?*}+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder})]}];; Filter by archived status(case include-archived-items:exclude [:= archived-column false]diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/table_utils.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/table_utils.cljindex 35316695c44..36a1dcdb2f2 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/table_utils.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/table_utils.clj@@ -44,19 +44,21 @@priority-tables []exclude-table-ids #{}}}](let [priority-table-ids (set (map :id priority-tables))+ {table-where-clause :clause table-cte :with} (mi/visible-filter-clause :model/Table+ :id+ {:user-id api/*current-user-id*+ :is-superuser? api/*is-superuser?*}+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder-and-native});; Fetch most viewed tables, excluding priority tables and excluded tablesfill-tables (t2/select [:model/Table :id :db_id :name :schema :description]- :db_id database-id- :active true+ :db_id database-id+ :active true:visibility_type nil- {:where (mi/visible-filter-clause :model/Table- :id- {:user-id api/*current-user-id*- :is-superuser? api/*is-superuser?*}- {:perms/view-data :unrestricted- :perms/create-queries :query-builder-and-native})- :order-by [[:view_count :desc]]- :limit all-tables-limit})+ (cond-> {:where table-where-clause+ :order-by [[:view_count :desc]]+ :limit all-tables-limit}+ table-cte (assoc :with table-cte)))fill-tables (remove #(or (priority-table-ids (:id %))(exclude-table-ids (:id %))) fill-tables)fill-tables (t2/hydrate fill-tables :fields)@@ -83,19 +85,21 @@priority-tables []exclude-table-ids #{}}}](let [priority-table-ids (set (map :id priority-tables))+ {table-where-clause :clause table-cte :with} (mi/visible-filter-clause :model/Table+ :id+ {:user-id api/*current-user-id*+ :is-superuser? api/*is-superuser?*}+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder-and-native});; Fetch most viewed tables, excluding priority tables and excluded tablesfill-tables (t2/select [:model/Table :id :db_id :name :schema :description]:db_id database-id:active true:visibility_type nil- {:where (mi/visible-filter-clause :model/Table- :id- {:user-id api/*current-user-id*- :is-superuser? api/*is-superuser?*}- {:perms/view-data :unrestricted- :perms/create-queries :query-builder-and-native})- :order-by [[:view_count :desc]]- :limit all-tables-limit})+ (cond-> {:where table-where-clause+ :order-by [[:view_count :desc]]+ :limit all-tables-limit}+ table-cte (assoc :with table-cte)))fill-tables (remove #(or (priority-table-ids (:id %))(exclude-table-ids (:id %))) fill-tables)all-tables (concat priority-tables fill-tables)@@ -164,12 +168,14 @@(defn- visible-filter-clause[]- (mi/visible-filter-clause :model/Table- :id- {:user-id api/*current-user-id*- :is-superuser? api/*is-superuser?*}- {:perms/view-data :unrestricted- :perms/create-queries :query-builder-and-native}))+ (let [{table-where-clause :clause table-cte :with} (mi/visible-filter-clause :model/Table+ :id+ {:user-id api/*current-user-id*+ :is-superuser? api/*is-superuser?*}+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder-and-native})]+ (cond-> {:where table-where-clause}+ table-cte (assoc :with table-cte))))(defn find-matching-tables"Find tables in the database that are similar to the unrecognized tables using fuzzy matching.@@ -186,6 +192,7 @@Returns:A vector of realized Table model instances that match the unrecognized tables."[database-id unrecognized-tables used-ids]+(into [](keep (fn [table](when (some #(matching-tables? table % {:match-schema? false}) unrecognized-tables)@@ -194,8 +201,8 @@:db_id database-id:active true:visibility_type nil- (cond-> {:where (visible-filter-clause)- :limit 10000}+ (cond-> (assoc (visible-filter-clause)+ :limit 10000)(seq used-ids) (update :where #(if %[:and % [:not-in :id used-ids]][:not-in :id used-ids]))))))@@ -212,7 +219,7 @@:id [:in table-ids]:active true:visibility_type nil- {:where (visible-filter-clause)})))+ (visible-filter-clause))))(defn used-tables"Return all tables used in the query, including fuzzy-matched ones.diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/index.cljindex 5292b45c93d..9004220c1c7 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/index.clj@@ -568,6 +568,24 @@(update base-query :where #(into [:and] [% filters]))base-query)))+(defn- flatten-ctes+ "Flatten nested :with clauses into a single top-level :with.+ PostgreSQL doesn't support nested CTEs, so we need to extract all nested :with clauses+ and put them at the top level.++ Takes a query map and returns {:ctes [...] :query query-without-with}"+ [query]+ (if-not (:with query)+ {:ctes [] :query query}+ (let [ctes (reduce+ (fn [acc [cte-name cte-query]]+ (let [{:keys [ctes query]} (flatten-ctes cte-query)]+ (into acc (conj ctes [cte-name query]))))+ []+ (:with query))]+ {:ctes ctes+ :query (dissoc query :with)})))+(defn- hybrid-select"For a given `col-name` return a :coalesce expression to reference it from the outer hybrid search query.@@ -600,8 +618,14 @@;; The purpose of this query is just to project the coalesced hybrid columns with standard names so the scorers know;; what to call them (e.g. :model rather than [:coalesced :v.model :t.model]). Likewise, the :search_index alias;; allows us to re-use scoring expressions between the appdb and semantic backends without adjusting column names.+ ;;+ ;; We flatten the nested CTEs because PostgreSQL doesn't support nested WITH clauses.+ ;; The hybrid-query contains nested :with clauses from semantic-search-query and keyword-search-query,+ ;; which we extract and place at the top level.(let [hybrid-query (hybrid-search-query index embedding search-context)- full-query {:with [[:hybrid_results hybrid-query]]+ {:keys [ctes query]} (flatten-ctes hybrid-query)+ all-ctes (conj ctes [:hybrid_results query])+ full-query {:with all-ctes:select [:id :model_id :model :content :verified :metadata :semantic_rank :keyword_rank]:from [[:hybrid_results :search_index]]:limit (semantic-settings/semantic-search-results-limit)}]diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.cljindex 46d32a1b309..c9ae08b0d8a 100644--- a/src/metabase/models/interface.clj+++ b/src/metabase/models/interface.clj@@ -674,10 +674,12 @@"Please consider adding one. See dox for `can-update?` for more details."))))(defmulti visible-filter-clause- "Return a honey SQL query fragment that will limit another query to only selecting records visible to the supplied user- by filtering on a supplied column or honeysql expression, using a the map of permission type->minimum permission-level.+ "Return a map with:+ - :clause - honey SQL WHERE clause fragment to filter records visible to the user+ - :with - optional vector of CTE definitions [[name query] ...] to be merged into the query- Defaults to returning a no-op false statement 0=1."+ Uses the map of permission type->minimum permission-level for filtering.+ Defaults to returning a no-op false statement {:clause [:= 0 1]}."{:arglists '([model column-or-exp user-info perm-type->perm-level])}dispatch-on-model)@@ -768,7 +770,7 @@(defmethod visible-filter-clause :default[_m _column-or-expression _user-info _perm-type->perm-level]- [:= [:inline 0] [:inline 1]])+ {:clause [:= [:inline 0] [:inline 1]]});;;; [[to-json]]diff --git a/src/metabase/permissions/core.clj b/src/metabase/permissions/core.cljindex 90fe3312e2c..574e84ab63c 100644--- a/src/metabase/permissions/core.clj+++ b/src/metabase/permissions/core.clj@@ -67,6 +67,7 @@PermissionMappingvisible-database-filter-selectvisible-table-filter-select+ visible-table-filter-with-cteselect-tables-and-groups-granting-perm][metabase.permissions.models.permissionsaudit-namespace-clausediff --git a/src/metabase/permissions/models/data_permissions/sql.clj b/src/metabase/permissions/models/data_permissions/sql.cljindex f8b50187d0f..726e6cf2b41 100644--- a/src/metabase/permissions/models/data_permissions/sql.clj+++ b/src/metabase/permissions/models/data_permissions/sql.clj@@ -75,17 +75,13 @@:least (perm-type-to-least-int-case perm-type :dp.perm_value))(perm-type-to-int-inline perm-type required-level)])-(mu/defn- user-in-group-half-join+(mu/defn- user-in-group-clause+ "Returns a simple IN clause to check if dp.group_id is in the user's groups.+ This is more efficient than nested EXISTS subqueries."[user-id :- pos-int?]- [:exists {:select [1]- :from [[:permissions_group :pg]]- :where [:and- [:= :pg.id :dp.group_id]- [:exists {:select [1]- :from [[:permissions_group_membership :pgm]]- :where [:and- [:= :pgm.group_id :pg.id]- [:= :pgm.user_id [:inline user-id]]]}]]}])+ [:in :dp.group_id {:select [:group_id]+ :from [:permissions_group_membership]+ :where [:= :user_id [:inline user-id]]}])(mu/defn- has-perms-for-table-as-honey-sql?"Builds an EXIST (SELECT ...) half-join to filter tables that a user has the required permissions for. It builds the subselect by as a@@ -103,7 +99,7 @@[:and [:= :mt.db_id :dp.db_id][:= :dp.table_id nil]][:= :mt.id :dp.table_id]]- (user-in-group-half-join user-id)]+ (user-in-group-clause user-id)]:group-by [:mt.id]:having [(perm-condition perm-type required-level (or most-or-least :least))]}])@@ -139,6 +135,91 @@(not (sequential? perm-level)) vector))]))permission-mapping))})+(mu/defn- permission-type-having-clause+ "Builds a HAVING clause condition for a single permission type using conditional aggregation."+ [perm-type :- :keyword+ required-level :- :keyword+ most-or-least :- [:enum :most :least]]+ (let [agg-fn (case most-or-least+ :most :max+ :least :min)+ ;; Build conditional: CASE WHEN dp.perm_type = ? THEN <int-case> END+ conditional-case [:case+ [:= :dp.perm_type (h2x/literal perm-type)]+ (perm-type-to-int-case perm-type :dp.perm_value)]]+ [(case most-or-least+ :most :>=+ :least :<=)+ (if (= perm-type :perms/view-data)+ ;; Special handling for view-data permission priority+ (let [minimum-perm-value [agg-fn+ [:case+ [:= :dp.perm_type (h2x/literal perm-type)]+ [:case+ [:= :dp.perm_value [:inline "unrestricted"]] [:inline 0]+ [:= :dp.perm_value [:inline "blocked"]] [:inline 1]+ [:= :dp.perm_value [:inline "legacy-no-self-service"]] [:inline 2]]]]]+ [:case+ [:= minimum-perm-value [:inline 0]] [:inline 0]+ [:= minimum-perm-value [:inline 1]] [:inline 2]+ [:= minimum-perm-value [:inline 2]] [:inline 1]])+ [agg-fn conditional-case])+ (perm-type-to-int-inline perm-type required-level)]))++(mu/defn visible-table-filter-with-cte+ "Returns a map with :with (CTE definitions) and :clause (WHERE clause fragment) for filtering+ tables visible to the user. Uses a CTE to compute permitted table IDs once rather than using+ correlated subqueries, which provides better performance for large numbers of tables.++ The returned map can be merged into a query by adding :with to the query's :with vector+ and using :clause in the WHERE clause.++ Uses UNION ALL to separate table-level and database-level permission lookups, avoiding+ inefficient BitmapOr scans that occur with OR joins."+ [column-or-exp :- :any+ {:keys [user-id is-superuser?]} :- UserInfo+ permission-mapping :- PermissionMapping]+ (if is-superuser?+ {:clause [:= [:inline 1] [:inline 1]]}+ (let [perm-types (keys permission-mapping)+ perm-type-filter (into [:or]+ (map (fn [pt] [:= :dp.perm_type (h2x/literal pt)])+ perm-types))+ having-conditions (into [:and]+ (map (fn [[perm-type perm-level]]+ (let [[level most-or-least] (if (sequential? perm-level)+ perm-level+ [perm-level :least])]+ (permission-type-having-clause perm-type level most-or-least)))+ permission-mapping))+ user-groups-clause (user-in-group-clause user-id)]+ {:with [;; First CTE: collect all permission grants that apply to each table+ [:table_permissions+ {:union-all+ [;; Table-level permissions (direct grant to table)+ {:select [:mt.id :dp.perm_type :dp.perm_value]+ :from [[:data_permissions :dp]]+ :join [[:metabase_table :mt] [:= :mt.id :dp.table_id]]+ :where [:and+ [:not= :dp.table_id nil]+ user-groups-clause+ perm-type-filter]}+ ;; Database-level permissions (applies to all tables in db)+ {:select [:mt.id :dp.perm_type :dp.perm_value]+ :from [[:data_permissions :dp]]+ :join [[:metabase_table :mt] [:= :mt.db_id :dp.db_id]]+ :where [:and+ [:= :dp.table_id nil]+ user-groups-clause+ perm-type-filter]}]}]+ ;; Second CTE: aggregate and filter by permission requirements+ [:permitted_tables+ {:select [:dp.id]+ :from [[:table_permissions :dp]]+ :group-by [:dp.id]+ :having having-conditions}]]+ :clause [:in column-or-exp {:select [:id] :from [:permitted_tables]}]})))+(mu/defn select-tables-and-groups-granting-perm"Selects table.id and the group.id of all permissions groups that give the provided user the provided permission level or apermission level either more or less restrictive than the supplied level."@@ -182,7 +263,7 @@:where [:and[:= :dp.perm_type (h2x/literal perm-type)][:= :md.id :dp.db_id]- (user-in-group-half-join user-id)]+ (user-in-group-clause user-id)]:group-by [:md.id]:having [(perm-condition perm-type required-level (or most-or-least :least))]}])diff --git a/src/metabase/search/appdb/core.clj b/src/metabase/search/appdb/core.cljindex 6ad0ffd9a26..12685ebd5aa 100644--- a/src/metabase/search/appdb/core.clj+++ b/src/metabase/search/appdb/core.clj@@ -75,20 +75,22 @@(update :last_edited_at parse-datetime)))(defn add-table-where-clauses- "Add a `WHERE` clause to the query to only return tables the current user has access to"+ "Add a `WHERE` clause to the query to only return tables the current user has access to.+ Also adds any CTEs required for permission filtering."[search-ctx qry]- (sql.helpers/where qry- [:or- [:= :search_index.model nil]- [:!= :search_index.model [:inline "table"]]- [:and- [:= :search_index.model [:inline "table"]]- [:exists {:select [1]- :from [[:metabase_table :mt_toplevel]]- :where [:and [:= :mt_toplevel.id [:cast :search_index.model_id (case (mdb/db-type)- :mysql :signed- :integer)]]- (search.permissions/permitted-tables-clause search-ctx :mt_toplevel.id)]}]]]))+ (let [model-id-col [:cast :search_index.model_id (case (mdb/db-type)+ :mysql :signed+ :integer)]+ {:keys [with clause]} (search.permissions/permitted-tables-clause search-ctx model-id-col)]+ (cond-> qry+ (seq with) (update :with (fnil into []) with)+ true (sql.helpers/where+ [:or+ [:= :search_index.model nil]+ [:!= :search_index.model [:inline "table"]]+ [:and+ [:= :search_index.model [:inline "table"]]+ clause]]))))(def null-collection-id-should-be-ignored-for-unpublished-tables-clause"For every model except tables, a null collection ID means the thing is in Our Analytics.diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.cljindex 91352cfa012..16bb55e05e3 100644--- a/src/metabase/search/in_place/legacy.clj+++ b/src/metabase/search/in_place/legacy.clj@@ -169,20 +169,26 @@query))(defn add-table-where-clauses- "Add a `WHERE` clause to the query to only return tables the current user has access to"+ "Add a `WHERE` clause to the query to only return tables the current user has access to.+ Also adds any CTEs required for permission filtering."[qry model search-ctx]- (sql.helpers/where qry (case model- "table" [:and- (search.permissions/permitted-tables-clause search-ctx :table.id)- [:or- [:not :is_published]- (search.permissions/permitted-collections-clause search-ctx :collection_id)]]- "search-index" [:or- [:= :search_index.model nil]- [:!= :search_index.model [:inline "table"]]- [:and- [:= :search_index.model [:inline "table"]]- (search.permissions/permitted-tables-clause search-ctx :search_index.model_id)]])))+ (let [col (case model "table" :table.id "search-index" :search_index.model_id)+ {:keys [with clause]} (search.permissions/permitted-tables-clause search-ctx col)]+ (cond-> qry+ (seq with) (update :with (fnil into []) with)+ true (sql.helpers/where+ (case model+ "table" [:and+ clause+ [:or+ [:not :is_published]+ (search.permissions/permitted-collections-clause search-ctx :collection_id)]]+ "search-index" [:or+ [:= :search_index.model nil]+ [:!= :search_index.model [:inline "table"]]+ [:and+ [:= :search_index.model [:inline "table"]]+ clause]])))))(mu/defn add-collection-join-and-where-clauses"Add a `WHERE` clause to the query to only return Collections the Current User has access to; join against Collection,@@ -619,15 +625,31 @@(sql.helpers/left-join [:collection :collection] [:and :table.is_published [:= :table.collection_id :collection.id]])(sql.helpers/left-join :metabase_database [:= :table.db_id :metabase_database.id]))))+(defn- extract-and-hoist-ctes+ "Extract :with clauses from a collection of queries and return a map with:+ - :ctes - all CTEs collected from queries (deduplicated by name)+ - :queries - queries with their :with clauses removed++ This is needed because MySQL/MariaDB doesn't support CTEs inside UNION ALL subqueries -+ the WITH clause must be at the outermost level of the statement."+ [queries]+ (let [all-ctes (into [] (comp (mapcat :with) (distinct)) queries)+ queries-without-ctes (mapv #(dissoc % :with) queries)]+ {:ctes all-ctes+ :queries queries-without-ctes}))+(defmethod search.engine/model-set :search.engine/in-place[search-ctx]- (let [model-queries (for [model (search.in-place.filter/search-context->applicable-models- ;; It's unclear why we don't use the existing :models- (assoc search-ctx :models search.config/all-models))]- {:nest (sql.helpers/limit (search-query-for-model model search-ctx) 1)})- query (when (pos-int? (count model-queries))- {:select [:*]- :from [[{:union-all model-queries} :dummy_alias]]})]+ (let [raw-queries (vec (for [model (search.in-place.filter/search-context->applicable-models+ ;; It's unclear why we don't use the existing :models+ (assoc search-ctx :models search.config/all-models))]+ (search-query-for-model model search-ctx)))+ {:keys [ctes queries]} (extract-and-hoist-ctes raw-queries)+ nested-queries (mapv #(hash-map :nest (sql.helpers/limit % 1)) queries)+ query (when (pos-int? (count nested-queries))+ (cond-> {:select [:*]+ :from [[{:union-all nested-queries} :dummy_alias]]}+ (seq ctes) (assoc :with ctes)))](into #{} (map :model) (some-> query mdb/query))))(mu/defn full-search-query@@ -645,13 +667,16 @@{:limit search.config/*db-max-results*}):else- {:select [:*]- :from [[{:union-all (vec (for [model models- :let [query (search-query-for-model model search-ctx)]- :when (seq query)]- query))} :alias_is_required_by_sql_but_not_needed_here]]- :order-by order-clause- :limit search.config/*db-max-results*})))+ (let [model-queries (vec (for [model models+ :let [query (search-query-for-model model search-ctx)]+ :when (seq query)]+ query))+ {:keys [ctes queries]} (extract-and-hoist-ctes model-queries)]+ (cond-> {:select [:*]+ :from [[{:union-all queries} :alias_is_required_by_sql_but_not_needed_here]]+ :order-by order-clause+ :limit search.config/*db-max-results*}+ (seq ctes) (assoc :with ctes))))));; Return a reducible-query corresponding to searching the entities without an index.(defn- resultsdiff --git a/src/metabase/search/permissions.clj b/src/metabase/search/permissions.cljindex fb293ef0040..de8b85fe044 100644--- a/src/metabase/search/permissions.clj+++ b/src/metabase/search/permissions.clj@@ -49,8 +49,9 @@[:= :collection.namespace "shared-tenant-collection"]]])(mu/defn permitted-tables-clause- "Build the WHERE clause corresponding to which tables the given user has access to."- [{:keys [current-user-id is-superuser?]} :- SearchContext table-id-col :- :keyword]+ "Build the WHERE clause and optional CTEs for table permission filtering.+ Returns a map with :clause (WHERE clause fragment) and :with (optional CTE definitions)."+ [{:keys [current-user-id is-superuser?]} :- SearchContext table-id-col :- [:or :keyword [:vector :keyword]]](mi/visible-filter-clause:model/Tabletable-id-coldiff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex 7eff6dbd0ca..51df676ef32 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -274,8 +274,7 @@column-or-exp :- :anyuser-info :- perms/UserInfopermission-mapping :- perms/PermissionMapping]- [:in column-or-exp- (perms/visible-table-filter-select :id user-info permission-mapping)])+ (perms/visible-table-filter-with-cte column-or-exp user-info permission-mapping));;; ------------------------------------------------ Serdes Hashing -------------------------------------------------diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 3e3ea5573f4..4eba736e07f 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -129,8 +129,8 @@(mu/defmethod mi/visible-filter-clause :model/Database[_model column-or-exp user-info permission-mapping]- [:in column-or-exp- (perms/visible-database-filter-select user-info permission-mapping)])+ {:clause [:in column-or-exp+ (perms/visible-database-filter-select user-info permission-mapping)]})(defn- infer-db-schedules"Infer database schedule settings based on its options."diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.cljindex 0cdf7ef2534..8bcdd6d18ae 100644--- a/src/metabase/warehouses_rest/api.clj+++ b/src/metabase/warehouses_rest/api.clj@@ -274,7 +274,7 @@[:= :router_database_id router-database-id][:= :router_database_id nil])]where-clause (if filter-by-data-access?- [:and base-where (mi/visible-filter-clause :model/Database :id user-info permission-mapping)]+ [:and base-where (:clause (mi/visible-filter-clause :model/Database :id user-info permission-mapping))]base-where)dbs (t2/select :model/Database {:order-by [:%lower.name :%lower.engine]:where where-clause})]diff --git a/test/metabase/warehouse_schema/models/table_test.clj b/test/metabase/warehouse_schema/models/table_test.cljindex 9714a21c249..8f2badd053a 100644--- a/test/metabase/warehouse_schema/models/table_test.clj+++ b/test/metabase/warehouse_schema/models/table_test.clj@@ -180,191 +180,316 @@(t2/hydrate :pk_field):pk_field))))-(deftest visible-filter-clause-table-level-test- (testing "visible-tables-filter-clause generates a HoneySQL clause that filters tables based on user permissions"+;;; ------------------------------------------ visible-filter-clause tests ------------------------------------------++(defn- fetch-visible-table-ids+ "Fetches visible table IDs using `visible-filter-clause` with the new `:clause/:with` return structure."+ [db-id user-info permission-mapping table-id-field]+ (let [{:keys [clause with]} (mi/visible-filter-clause :model/Table table-id-field user-info permission-mapping)]+ (t2/select-pks-set [:model/Table]+ (cond-> {:where [:and [:= :db_id db-id] clause]}+ with (assoc :with with)))))++(defn- superuser-info+ "Returns user-info map for a superuser."+ [user-id]+ {:user-id user-id+ :is-superuser? true})++(defn- regular-user-info+ "Returns user-info map for a non-superuser."+ [user-id]+ {:user-id user-id+ :is-superuser? false})++(deftest visible-filter-clause-superuser-sees-all-tables-test+ (testing "Superuser should see all tables regardless of permissions"(mt/with-no-data-perms-for-all-users!- (mt/with-temp [:model/Database {db-id :id} {}- :model/Table {table-view-data-unrestricted-create-queries-query-builder-id :id} {:db_id db-id :name "Table1"}- :model/Table {table-view-data-unrestricted-create-queries-query-builder-and-native-id :id} {:db_id db-id :name "Table2"}- :model/Table {table-view-data-blocked-create-queries-no-id :id} {:db_id db-id :name "Table3"}- :model/Table {table-view-data-legacy-no-self-service-create-queries-no-id :id} {:db_id db-id :name "Table4"}- :model/PermissionsGroup pg1 {}- :model/PermissionsGroup pg2 {}]-+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}+ :model/Table {table-3 :id} {:db_id db-id :name "Table3"}+ :model/PermissionsGroup pg {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg)+ (t2/delete! :model/DataPermissions :db_id db-id)+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)+ (is (= #{table-1 table-2 table-3}+ (fetch-visible-table-ids db-id+ (superuser-info (mt/user->id :rasta))+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder}+ :id)))))))++(deftest visible-filter-clause-filters-by-view-and-query-perms-test+ (testing "Non-superuser sees only tables where they have both view and query permissions"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}+ :model/Table {table-blocked :id} {:db_id db-id :name "Table3"}+ :model/PermissionsGroup pg {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg)+ (t2/delete! :model/DataPermissions :db_id db-id)+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)+ (data-perms/set-table-permission! pg table-1 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg table-1 :perms/create-queries :query-builder)+ (data-perms/set-table-permission! pg table-2 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg table-2 :perms/create-queries :query-builder-and-native)+ (data-perms/set-table-permission! pg table-blocked :perms/view-data :blocked)+ (data-perms/set-table-permission! pg table-blocked :perms/create-queries :no)+ (is (= #{table-1 table-2}+ (fetch-visible-table-ids db-id+ (regular-user-info (mt/user->id :rasta))+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder}+ :id)))))))++(deftest visible-filter-clause-coalesce-expression-test+ (testing "Filter clause works with coalesce expression for table ID"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}+ :model/PermissionsGroup pg {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg)+ (t2/delete! :model/DataPermissions :db_id db-id)+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)+ (data-perms/set-table-permission! pg table-1 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg table-1 :perms/create-queries :query-builder)+ (data-perms/set-table-permission! pg table-2 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg table-2 :perms/create-queries :query-builder-and-native)+ (is (= #{table-1 table-2}+ (fetch-visible-table-ids db-id+ (regular-user-info (mt/user->id :rasta))+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder}+ [:coalesce :id :metabase_table.id])))))))++(deftest visible-filter-clause-qualified-keyword-test+ (testing "Filter clause works with qualified keyword for table ID"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}+ :model/PermissionsGroup pg {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg)+ (t2/delete! :model/DataPermissions :db_id db-id)+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)+ (data-perms/set-table-permission! pg table-1 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg table-1 :perms/create-queries :query-builder)+ (data-perms/set-table-permission! pg table-2 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg table-2 :perms/create-queries :query-builder-and-native)+ (is (= #{table-1 table-2}+ (fetch-visible-table-ids db-id+ (regular-user-info (mt/user->id :rasta))+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder}+ :metabase_table.id)))))))++(deftest visible-filter-clause-view-data-only-test+ (testing "Non-superuser requiring only :view-data :unrestricted sees tables with that permission"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}+ :model/Table {table-blocked :id} {:db_id db-id :name "Table3"}+ :model/PermissionsGroup pg {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg)+ (t2/delete! :model/DataPermissions :db_id db-id)+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)+ (data-perms/set-table-permission! pg table-1 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg table-1 :perms/create-queries :query-builder)+ (data-perms/set-table-permission! pg table-2 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg table-2 :perms/create-queries :query-builder-and-native)+ (data-perms/set-table-permission! pg table-blocked :perms/view-data :blocked)+ (data-perms/set-table-permission! pg table-blocked :perms/create-queries :no)+ (is (= #{table-1 table-2}+ (fetch-visible-table-ids db-id+ (regular-user-info (mt/user->id :rasta))+ {:perms/view-data :unrestricted+ :perms/create-queries :no}+ :id)))))))++(deftest visible-filter-clause-legacy-no-self-service-test+ (testing "Non-superuser requiring :view-data :legacy-no-self-service sees tables at that level or above"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}+ :model/Table {table-blocked :id} {:db_id db-id :name "Table3"}+ :model/Table {table-legacy :id} {:db_id db-id :name "Table4"}+ :model/PermissionsGroup pg {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg)+ (t2/delete! :model/DataPermissions :db_id db-id)+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)+ (data-perms/set-table-permission! pg table-1 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg table-1 :perms/create-queries :query-builder)+ (data-perms/set-table-permission! pg table-2 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg table-2 :perms/create-queries :query-builder-and-native)+ (data-perms/set-table-permission! pg table-blocked :perms/view-data :blocked)+ (data-perms/set-table-permission! pg table-blocked :perms/create-queries :no)+ (data-perms/set-table-permission! pg table-legacy :perms/view-data :legacy-no-self-service)+ (data-perms/set-table-permission! pg table-legacy :perms/create-queries :no)+ (is (= #{table-1 table-2 table-legacy}+ (fetch-visible-table-ids db-id+ (regular-user-info (mt/user->id :rasta))+ {:perms/view-data :legacy-no-self-service+ :perms/create-queries :no}+ :id)))))))++(deftest visible-filter-clause-blocked-level-sees-all-test+ (testing "Non-superuser requiring :view-data :blocked sees all tables since all values are more permissive"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}+ :model/Table {table-blocked :id} {:db_id db-id :name "Table3"}+ :model/Table {table-legacy :id} {:db_id db-id :name "Table4"}+ :model/PermissionsGroup pg {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg)+ (t2/delete! :model/DataPermissions :db_id db-id)+ (data-perms/set-database-permission! pg db-id :perms/view-data :blocked)+ (data-perms/set-database-permission! pg db-id :perms/create-queries :no)+ (data-perms/set-table-permission! pg table-1 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg table-1 :perms/create-queries :query-builder)+ (data-perms/set-table-permission! pg table-2 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg table-2 :perms/create-queries :query-builder-and-native)+ (data-perms/set-table-permission! pg table-blocked :perms/view-data :blocked)+ (data-perms/set-table-permission! pg table-blocked :perms/create-queries :no)+ (data-perms/set-table-permission! pg table-legacy :perms/view-data :legacy-no-self-service)+ (data-perms/set-table-permission! pg table-legacy :perms/create-queries :no)+ (is (= #{table-1 table-2 table-blocked table-legacy}+ (fetch-visible-table-ids db-id+ (regular-user-info (mt/user->id :rasta))+ {:perms/view-data :blocked+ :perms/create-queries :no}+ :id)))))))++(deftest visible-filter-clause-blocked-takes-precedence-test+ (testing "Blocked permission takes precedence over legacy-no-self-service across groups"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}+ :model/Table {table-legacy :id} {:db_id db-id :name "Table4"}+ :model/PermissionsGroup pg1 {}+ :model/PermissionsGroup pg2 {}](perms/add-user-to-group! (mt/user->id :rasta) pg1)(perms/add-user-to-group! (mt/user->id :rasta) pg2)-(t2/delete! :model/DataPermissions :db_id db-id)-+ ;; pg1 has legacy-no-self-service for table-legacy(data-perms/set-database-permission! pg1 db-id :perms/view-data :blocked)(data-perms/set-database-permission! pg1 db-id :perms/create-queries :no)-+ (data-perms/set-table-permission! pg1 table-1 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg1 table-1 :perms/create-queries :query-builder)+ (data-perms/set-table-permission! pg1 table-2 :perms/view-data :unrestricted)+ (data-perms/set-table-permission! pg1 table-2 :perms/create-queries :query-builder-and-native)+ (data-perms/set-table-permission! pg1 table-legacy :perms/view-data :legacy-no-self-service)+ (data-perms/set-table-permission! pg1 table-legacy :perms/create-queries :no)+ ;; pg2 blocks table-legacy, which should take precedence(data-perms/set-database-permission! pg2 db-id :perms/view-data :legacy-no-self-service)(data-perms/set-database-permission! pg2 db-id :perms/create-queries :no)-- (data-perms/set-table-permission! pg1 table-view-data-unrestricted-create-queries-query-builder-id :perms/view-data :unrestricted)- (data-perms/set-table-permission! pg1 table-view-data-unrestricted-create-queries-query-builder-id :perms/create-queries :query-builder)-- (data-perms/set-table-permission! pg1 table-view-data-unrestricted-create-queries-query-builder-and-native-id :perms/view-data :unrestricted)- (data-perms/set-table-permission! pg1 table-view-data-unrestricted-create-queries-query-builder-and-native-id :perms/create-queries :query-builder-and-native)-- (data-perms/set-table-permission! pg1 table-view-data-blocked-create-queries-no-id :perms/view-data :blocked)- (data-perms/set-table-permission! pg1 table-view-data-blocked-create-queries-no-id :perms/create-queries :no)-- (data-perms/set-table-permission! pg1 table-view-data-legacy-no-self-service-create-queries-no-id :perms/view-data :legacy-no-self-service)- (data-perms/set-table-permission! pg1 table-view-data-legacy-no-self-service-create-queries-no-id :perms/create-queries :no)-- (let [user-id (mt/user->id :rasta)- fetch-visible-ids (fn [user-info permission-mapping table-id-field]- (let [filter-clause (mi/visible-filter-clause :model/Table table-id-field user-info permission-mapping)]- (t2/select-pks-set [:model/Table]- {:where- [:and- [:= :db_id db-id]- filter-clause]})))]-- (testing "Superuser should see all tables"- (let [user-info {:user-id user-id ; User ID doesn't matter for superuser- :is-superuser? true}- permission-map {:perms/view-data :unrestricted ; Levels don't matter for superuser- :perms/create-queries :query-builder}]- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id- table-view-data-unrestricted-create-queries-query-builder-and-native-id- table-view-data-blocked-create-queries-no-id- table-view-data-legacy-no-self-service-create-queries-no-id}- (fetch-visible-ids user-info permission-map :id)))))-- (testing "Non-superuser should only see tables where they have both view and query permissions (mimics mi/can-read?)"- (let [user-info {:user-id user-id- :is-superuser? false}- permission-map {:perms/view-data :unrestricted- :perms/create-queries :query-builder}]- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id- table-view-data-unrestricted-create-queries-query-builder-and-native-id}- (fetch-visible-ids user-info permission-map :id))- "Clause should filter out tables without sufficient view/query permissions")-- (testing "using a sequence of fields for the table ID"- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id- table-view-data-unrestricted-create-queries-query-builder-and-native-id}- (fetch-visible-ids user-info permission-map [:coalesce :id :metabase_table.id]))- "Clause should work with coalesce when a sequence of fields is provided"))-- (testing "using a qualified keyword"- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id- table-view-data-unrestricted-create-queries-query-builder-and-native-id}- (fetch-visible-ids user-info permission-map :metabase_table.id))- "Clause should work with qualified keyword"))))-- (testing "Non-superuser requiring only :view-data :unrestricted should see tables 1 and 2"- (let [user-info {:user-id user-id- :is-superuser? false}- permission-map {:perms/view-data :unrestricted- :perms/create-queries :no}]- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id- table-view-data-unrestricted-create-queries-query-builder-and-native-id}- (fetch-visible-ids user-info permission-map :id))- "Clause should respect the provided permission levels")))-- (testing "Non-superuser requiring only :view-data :legacy-no-self-service should see tables 1, 2, and 4"- (let [user-info {:user-id user-id- :is-superuser? false}- permission-map {:perms/view-data :legacy-no-self-service- :perms/create-queries :no}]- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id- table-view-data-unrestricted-create-queries-query-builder-and-native-id- table-view-data-legacy-no-self-service-create-queries-no-id}- (fetch-visible-ids user-info permission-map :id))- "Clause should respect the provided permission levels")))-- (testing "Non-superuser requiring :view-data :blocked should see all tables since all values are more permissive "- (let [user-info {:user-id user-id- :is-superuser? false}- permission-map {:perms/view-data :blocked- :perms/create-queries :no}]- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id- table-view-data-unrestricted-create-queries-query-builder-and-native-id- table-view-data-blocked-create-queries-no-id- table-view-data-legacy-no-self-service-create-queries-no-id}- (fetch-visible-ids user-info permission-map :id))- "Clause should filter correctly when requiring :blocked level")))-- (testing "Blocked table takes precedence over legacy-no-self-service when in any group a user is a member of"- (data-perms/set-table-permission! pg2 table-view-data-legacy-no-self-service-create-queries-no-id :perms/view-data :blocked)- (data-perms/set-table-permission! pg2 table-view-data-legacy-no-self-service-create-queries-no-id :perms/create-queries :no)-- (let [user-info {:user-id user-id- :is-superuser? false}- permission-map {:perms/view-data :legacy-no-self-service- :perms/create-queries :no}]- (is (= #{table-view-data-unrestricted-create-queries-query-builder-id- table-view-data-unrestricted-create-queries-query-builder-and-native-id}- (fetch-visible-ids user-info permission-map :id))- "Clause should respect the provided permission levels"))))))))--(deftest visible-filter-clause-db-level-test- (testing "visible-tables-filter-clause generates a HoneySQL clause that filters tables based on user permissions when only db level perms are set"- (mt/with-temp [:model/Database {db-id :id} {}- :model/Table {table-id-1 :id} {:db_id db-id :name "Table1"}- :model/Table {table-id-2 :id} {:db_id db-id :name "Table2"}- :model/Table {table-id-3 :id} {:db_id db-id :name "Table3"}]-+ (data-perms/set-table-permission! pg2 table-legacy :perms/view-data :blocked)+ (data-perms/set-table-permission! pg2 table-legacy :perms/create-queries :no)+ (is (= #{table-1 table-2}+ (fetch-visible-table-ids db-id+ (regular-user-info (mt/user->id :rasta))+ {:perms/view-data :legacy-no-self-service+ :perms/create-queries :no}+ :id)))))))++;;; ---------------------------------------- DB-level permissions tests ----------------------------------------++(deftest visible-filter-clause-db-level-superuser-test+ (testing "Superuser sees all tables with DB-level permissions"+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}+ :model/Table {table-3 :id} {:db_id db-id :name "Table3"}](data-perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)(data-perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries :no)-- (let [user-id (mt/user->id :rasta)- fetch-visible-ids (fn [user-cfg permission-map table-id-field]- (let [filter-clause (mi/visible-filter-clause :model/Table table-id-field user-cfg permission-map)]- (t2/select-pks-set [:model/Table]- {:where- [:and- [:= :db_id db-id]- filter-clause]})))]-- (testing "Superuser should see all tables"- (let [superuser-cfg {:user-id user-id ; User ID doesn't matter for superuser- :is-superuser? true}- permission-map {:perms/view-data :unrestricted ; Levels don't matter for superuser- :perms/create-queries :query-builder}]- (is (= #{table-id-1 table-id-2 table-id-3}- (fetch-visible-ids superuser-cfg permission-map :id)))))-- (testing "Non-superuser should only see tables where they have both view and query permissions (mimics mi/can-read?)"- (let [user-info {:user-id user-id- :is-superuser? false}- permission-map {:perms/view-data :unrestricted- :perms/create-queries :query-builder}]- (is (empty? ; Expecting empty set if no tables are visible with these perms- (fetch-visible-ids user-info permission-map :id))- "Clause should filter out tables without sufficient view/query permissions")-- (testing "using an expression for the table ID"- (is (empty?- (fetch-visible-ids user-info permission-map [:coalesce :id :metabase_table.id]))- "Clause should work with coalesce when a sequence of fields is provided"))-- (testing "using a qualified keyword"- (is (empty?- (fetch-visible-ids user-info permission-map :metabase_table.id))- "Clause should work with qualified keyword"))))-- (testing "Non-superuser requiring only :view-data :unrestricted should see tables 1, 2 and 3"- (let [user-info {:user-id user-id- :is-superuser? false}- permission-map {:perms/view-data :unrestricted- :perms/create-queries :no}]- (is (= #{table-id-1 table-id-2 table-id-3}- (fetch-visible-ids user-info permission-map :id))- "Clause should respect the provided permission levels")))-- (testing "Non-superuser requiring :view-data :blocked should all tables since all values are more permissive "- (let [user-info {:user-id user-id- :is-superuser? false}- permission-map {:perms/view-data :blocked- :perms/create-queries :no}]- (is (= #{table-id-1 table-id-2 table-id-3}- (fetch-visible-ids user-info permission-map :id))- "Clause should filter correctly when requiring :blocked level")))))))+ (is (= #{table-1 table-2 table-3}+ (fetch-visible-table-ids db-id+ (superuser-info (mt/user->id :rasta))+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder}+ :id))))))++(deftest visible-filter-clause-db-level-no-query-perms-test+ (testing "Non-superuser with DB-level view but no query perms sees no tables when both are required"+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {_table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {_table-2 :id} {:db_id db-id :name "Table2"}+ :model/Table {_table-3 :id} {:db_id db-id :name "Table3"}]+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries :no)+ (is (empty? (fetch-visible-table-ids db-id+ (regular-user-info (mt/user->id :rasta))+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder}+ :id))))))++(deftest visible-filter-clause-db-level-coalesce-test+ (testing "Filter clause with coalesce works for DB-level permissions"+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {_table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {_table-2 :id} {:db_id db-id :name "Table2"}]+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries :no)+ (is (empty? (fetch-visible-table-ids db-id+ (regular-user-info (mt/user->id :rasta))+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder}+ [:coalesce :id :metabase_table.id]))))))++(deftest visible-filter-clause-db-level-qualified-keyword-test+ (testing "Filter clause with qualified keyword works for DB-level permissions"+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {_table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {_table-2 :id} {:db_id db-id :name "Table2"}]+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries :no)+ (is (empty? (fetch-visible-table-ids db-id+ (regular-user-info (mt/user->id :rasta))+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder}+ :metabase_table.id))))))++(deftest visible-filter-clause-db-level-view-only-test+ (testing "Non-superuser requiring only :view-data :unrestricted sees all tables with DB-level unrestricted"+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}+ :model/Table {table-3 :id} {:db_id db-id :name "Table3"}]+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries :no)+ (is (= #{table-1 table-2 table-3}+ (fetch-visible-table-ids db-id+ (regular-user-info (mt/user->id :rasta))+ {:perms/view-data :unrestricted+ :perms/create-queries :no}+ :id))))))++(deftest visible-filter-clause-db-level-blocked-test+ (testing "Non-superuser requiring :blocked sees all tables since all values are more permissive"+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Table {table-1 :id} {:db_id db-id :name "Table1"}+ :model/Table {table-2 :id} {:db_id db-id :name "Table2"}+ :model/Table {table-3 :id} {:db_id db-id :name "Table3"}]+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! (perms-group/all-users) db-id :perms/create-queries :no)+ (is (= #{table-1 table-2 table-3}+ (fetch-visible-table-ids db-id+ (regular-user-info (mt/user->id :rasta))+ {:perms/view-data :blocked+ :perms/create-queries :no}+ :id))))))(deftest prevent-metabase-transform-data-source-change-test(testing "Cannot change data_source from metabase-transform"diff --git a/test/metabase/warehouses/models/database_test.clj b/test/metabase/warehouses/models/database_test.cljindex e6aba937912..44034658745 100644--- a/test/metabase/warehouses/models/database_test.clj+++ b/test/metabase/warehouses/models/database_test.clj@@ -620,8 +620,24 @@:tables(#(map :name %))))))-(deftest visible-filter-clause-test- (testing "Database visible-filter-clause generates a HoneySQL clause that filters databases based on user permissions"+;;; ---------------------------------------- visible-filter-clause tests ----------------------------------------++(defn- fetch-visible-db-ids+ "Helper to fetch visible database IDs using visible-filter-clause.+ Handles the :with/:clause return value format."+ [db-ids user-info permission-mapping column-field]+ (let [{:keys [clause]} (mi/visible-filter-clause :model/Database column-field user-info permission-mapping)]+ (t2/select-pks-set :model/Database+ {:where [:and+ clause+ [:in :id db-ids]]})))++(def ^:private default-permission-mapping+ {:perms/view-data :unrestricted+ :perms/create-queries :query-builder})++(deftest visible-filter-clause-superuser-test+ (testing "Superuser should see all databases"(mt/with-no-data-perms-for-all-users!(mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}:model/Database {db2-id :id} {:name "Database 2"}@@ -631,130 +647,204 @@:model/Table _ {:db_id db3-id :name "Table3"}:model/PermissionsGroup pg1 {}:model/PermissionsGroup pg2 {}]-- ;; Set up permissions: user has access to db1 and db2 via different groups, but not db3(perms/add-user-to-group! (mt/user->id :rasta) pg1)(perms/add-user-to-group! (mt/user->id :rasta) pg2)-- ;; Clear existing permissions for our test databases only(t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])-- ;; Grant permissions to specific databases(data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)(data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)-(data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)(data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)-- ;; No permissions for db3(data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)(data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)-- (let [user-id (mt/user->id :rasta)- fetch-visible-ids (fn [user-info permission-mapping column-field]- (let [filter-clause (mi/visible-filter-clause :model/Database column-field user-info permission-mapping)]- (t2/select-pks-set :model/Database- {:where [:and- filter-clause- [:in :id [db1-id db2-id db3-id]]]})))] ; Only check our test databases-- (testing "Superuser should see all databases"- (let [user-info {:user-id user-id :is-superuser? true}- permission-mapping {:perms/view-data :unrestricted- :perms/create-queries :query-builder}]- (is (= #{db1-id db2-id db3-id}- (fetch-visible-ids user-info permission-mapping :id))- "Superuser should see all databases")))-- (testing "Non-superuser should only see databases they have permissions for"- (let [user-info {:user-id user-id :is-superuser? false}- permission-mapping {:perms/view-data :unrestricted- :perms/create-queries :query-builder}]- (is (= #{db1-id db2-id}- (fetch-visible-ids user-info permission-mapping :id))- "Non-superuser should only see databases with sufficient permissions")))-- (testing "Using qualified column name"- (let [user-info {:user-id user-id :is-superuser? false}- permission-mapping {:perms/view-data :unrestricted- :perms/create-queries :query-builder}]- (is (= #{db1-id db2-id}- (fetch-visible-ids user-info permission-mapping :metabase_database.id))- "Should work with qualified column names")))-- (testing "Using column expression"- (let [user-info {:user-id user-id :is-superuser? false}- permission-mapping {:perms/view-data :unrestricted- :perms/create-queries :query-builder}]- (is (= #{db1-id db2-id}- (fetch-visible-ids user-info permission-mapping [:coalesce :id :metabase_database.id]))- "Should work with column expressions")))-- (testing "Different permission levels"- (testing "Requiring only view-data permissions"- (let [user-info {:user-id user-id :is-superuser? false}- permission-mapping {:perms/view-data :unrestricted- :perms/create-queries :no}]- (is (= #{db1-id db2-id}- (fetch-visible-ids user-info permission-mapping :id))- "Should include databases where user has view permissions")))-- (testing "Requiring blocked level permissions (most permissive)"- (let [user-info {:user-id user-id :is-superuser? false}- permission-mapping {:perms/view-data :blocked- :perms/create-queries :no}]- (is (= #{db1-id db2-id db3-id}- (fetch-visible-ids user-info permission-mapping :id))- "Should include all databases when requiring only blocked level"))))-- (testing "User with no permissions"- ;; Remove user from groups we added (avoid touching All Users group)- (t2/delete! :model/PermissionsGroupMembership- :user_id user-id- :group_id [:in [(:id pg1) (:id pg2)]])-- (let [user-info {:user-id user-id :is-superuser? false}- permission-mapping {:perms/view-data :unrestricted- :perms/create-queries :query-builder}]- (is (empty? (fetch-visible-ids user-info permission-mapping :id))- "User with no group memberships should see no databases"))))))))+ (is (= #{db1-id db2-id db3-id}+ (fetch-visible-db-ids [db1-id db2-id db3-id]+ {:user-id (mt/user->id :rasta) :is-superuser? true}+ default-permission-mapping+ :id)))))))++(deftest visible-filter-clause-non-superuser-test+ (testing "Non-superuser should only see databases they have permissions for"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}+ :model/Database {db2-id :id} {:name "Database 2"}+ :model/Database {db3-id :id} {:name "Database 3"}+ :model/Table _ {:db_id db1-id :name "Table1"}+ :model/Table _ {:db_id db2-id :name "Table2"}+ :model/Table _ {:db_id db3-id :name "Table3"}+ :model/PermissionsGroup pg1 {}+ :model/PermissionsGroup pg2 {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg1)+ (perms/add-user-to-group! (mt/user->id :rasta) pg2)+ (t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])+ (data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)+ (data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)+ (data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)+ (data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)+ (is (= #{db1-id db2-id}+ (fetch-visible-db-ids [db1-id db2-id db3-id]+ {:user-id (mt/user->id :rasta) :is-superuser? false}+ default-permission-mapping+ :id)))))))++(deftest visible-filter-clause-qualified-column-test+ (testing "Should work with qualified column names"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}+ :model/Database {db2-id :id} {:name "Database 2"}+ :model/Database {db3-id :id} {:name "Database 3"}+ :model/Table _ {:db_id db1-id :name "Table1"}+ :model/Table _ {:db_id db2-id :name "Table2"}+ :model/Table _ {:db_id db3-id :name "Table3"}+ :model/PermissionsGroup pg1 {}+ :model/PermissionsGroup pg2 {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg1)+ (perms/add-user-to-group! (mt/user->id :rasta) pg2)+ (t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])+ (data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)+ (data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)+ (data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)+ (data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)+ (is (= #{db1-id db2-id}+ (fetch-visible-db-ids [db1-id db2-id db3-id]+ {:user-id (mt/user->id :rasta) :is-superuser? false}+ default-permission-mapping+ :metabase_database.id)))))))++(deftest visible-filter-clause-column-expression-test+ (testing "Should work with column expressions"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}+ :model/Database {db2-id :id} {:name "Database 2"}+ :model/Database {db3-id :id} {:name "Database 3"}+ :model/Table _ {:db_id db1-id :name "Table1"}+ :model/Table _ {:db_id db2-id :name "Table2"}+ :model/Table _ {:db_id db3-id :name "Table3"}+ :model/PermissionsGroup pg1 {}+ :model/PermissionsGroup pg2 {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg1)+ (perms/add-user-to-group! (mt/user->id :rasta) pg2)+ (t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])+ (data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)+ (data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)+ (data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)+ (data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)+ (is (= #{db1-id db2-id}+ (fetch-visible-db-ids [db1-id db2-id db3-id]+ {:user-id (mt/user->id :rasta) :is-superuser? false}+ default-permission-mapping+ [:coalesce :id :metabase_database.id])))))))++(deftest visible-filter-clause-view-data-only-test+ (testing "Requiring only view-data permissions should include databases where user has view permissions"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}+ :model/Database {db2-id :id} {:name "Database 2"}+ :model/Database {db3-id :id} {:name "Database 3"}+ :model/Table _ {:db_id db1-id :name "Table1"}+ :model/Table _ {:db_id db2-id :name "Table2"}+ :model/Table _ {:db_id db3-id :name "Table3"}+ :model/PermissionsGroup pg1 {}+ :model/PermissionsGroup pg2 {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg1)+ (perms/add-user-to-group! (mt/user->id :rasta) pg2)+ (t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])+ (data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)+ (data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)+ (data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)+ (data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)+ (is (= #{db1-id db2-id}+ (fetch-visible-db-ids [db1-id db2-id db3-id]+ {:user-id (mt/user->id :rasta) :is-superuser? false}+ {:perms/view-data :unrestricted+ :perms/create-queries :no}+ :id)))))))++(deftest visible-filter-clause-blocked-level-test+ (testing "Requiring blocked level permissions (most permissive) should include all databases"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}+ :model/Database {db2-id :id} {:name "Database 2"}+ :model/Database {db3-id :id} {:name "Database 3"}+ :model/Table _ {:db_id db1-id :name "Table1"}+ :model/Table _ {:db_id db2-id :name "Table2"}+ :model/Table _ {:db_id db3-id :name "Table3"}+ :model/PermissionsGroup pg1 {}+ :model/PermissionsGroup pg2 {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg1)+ (perms/add-user-to-group! (mt/user->id :rasta) pg2)+ (t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])+ (data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)+ (data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)+ (data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)+ (data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)+ (is (= #{db1-id db2-id db3-id}+ (fetch-visible-db-ids [db1-id db2-id db3-id]+ {:user-id (mt/user->id :rasta) :is-superuser? false}+ {:perms/view-data :blocked+ :perms/create-queries :no}+ :id)))))))++(deftest visible-filter-clause-no-permissions-test+ (testing "User with no group memberships should see no databases"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-temp [:model/Database {db1-id :id} {:name "Database 1"}+ :model/Database {db2-id :id} {:name "Database 2"}+ :model/Database {db3-id :id} {:name "Database 3"}+ :model/Table _ {:db_id db1-id :name "Table1"}+ :model/Table _ {:db_id db2-id :name "Table2"}+ :model/Table _ {:db_id db3-id :name "Table3"}+ :model/PermissionsGroup pg1 {}+ :model/PermissionsGroup pg2 {}]+ (perms/add-user-to-group! (mt/user->id :rasta) pg1)+ (perms/add-user-to-group! (mt/user->id :rasta) pg2)+ (t2/delete! :model/DataPermissions :db_id [:in [db1-id db2-id db3-id]])+ (data-perms/set-database-permission! pg1 db1-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! pg1 db1-id :perms/create-queries :query-builder)+ (data-perms/set-database-permission! pg2 db2-id :perms/view-data :unrestricted)+ (data-perms/set-database-permission! pg2 db2-id :perms/create-queries :query-builder)+ (data-perms/set-database-permission! pg1 db3-id :perms/view-data :blocked)+ (data-perms/set-database-permission! pg2 db3-id :perms/view-data :blocked)+ ;; Remove user from groups we added (avoid touching All Users group)+ (t2/delete! :model/PermissionsGroupMembership+ :user_id (mt/user->id :rasta)+ :group_id [:in [(:id pg1) (:id pg2)]])+ (is (empty? (fetch-visible-db-ids [db1-id db2-id db3-id]+ {:user-id (mt/user->id :rasta) :is-superuser? false}+ default-permission-mapping+ :id)))))))(deftest visible-filter-clause-table-level-permissions-test- (testing "Database visible-filter-clause with table-level permissions"+ (testing "Database should be visible when user has access to at least one table"(mt/with-no-data-perms-for-all-users!(mt/with-temp [:model/Database {db-id :id} {:name "Test Database"}:model/Table {table1-id :id} {:db_id db-id :name "Table1"}:model/Table {table2-id :id} {:db_id db-id :name "Table2"}:model/Table _ {:db_id db-id :name "Table3"}:model/PermissionsGroup pg {}]-(perms/add-user-to-group! (mt/user->id :rasta) pg)-;; Clear existing permissions for our test database only(t2/delete! :model/DataPermissions :db_id db-id)-;; Block database-level access(data-perms/set-database-permission! pg db-id :perms/view-data :blocked)(data-perms/set-database-permission! pg db-id :perms/create-queries :no)-- ;; Grant table-level permissions to only table1 and table2+ ;; Grant table-level permissions to only table1 and table2 (table3 remains blocked)(data-perms/set-table-permission! pg table1-id :perms/view-data :unrestricted)(data-perms/set-table-permission! pg table1-id :perms/create-queries :query-builder)-(data-perms/set-table-permission! pg table2-id :perms/view-data :unrestricted)(data-perms/set-table-permission! pg table2-id :perms/create-queries :query-builder)- ;; table3 remains blocked-- (let [user-id (mt/user->id :rasta)- user-info {:user-id user-id :is-superuser? false}- permission-mapping {:perms/view-data :unrestricted- :perms/create-queries :query-builder}- filter-clause (mi/visible-filter-clause :model/Database :id user-info permission-mapping)- visible-db-ids (t2/select-pks-set :model/Database- {:where [:and- filter-clause- [:= :id db-id]]})] ; Only check our test database-- (is (contains? visible-db-ids db-id)- "Database should be visible when user has access to at least one table"))))))+ (is (contains? (fetch-visible-db-ids [db-id]+ {:user-id (mt/user->id :rasta) :is-superuser? false}+ default-permission-mapping+ :id)+ db-id))))))