The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Implement current-user-table-privileges for Oracle and SQL Server to fix N+1 query (#72043)
bryan9a2d08aec6c7
diff --git a/modules/drivers/oracle/src/metabase/driver/oracle.clj b/modules/drivers/oracle/src/metabase/driver/oracle.cljindex 13d2ed21b35..abac4b2b2ab 100644--- a/modules/drivers/oracle/src/metabase/driver/oracle.clj+++ b/modules/drivers/oracle/src/metabase/driver/oracle.clj@@ -62,7 +62,8 @@:now true;; these don't seem to ERROR on Oracle but they don't work as expected either, see;; https://github.com/metabase/metabase/pull/66982#issuecomment-3667113995- :regex/lookaheads-and-lookbehinds false}]+ :regex/lookaheads-and-lookbehinds false+ :table-privileges true}](defmethod driver/database-supports? [:oracle feature] [_driver _feature _db] supported?))(mr/def ::details@@ -762,3 +763,53 @@(defmethod driver/llm-sql-dialect-resource :oracle [_]"metabot/prompts/dialects/oracle.md")++(defmethod sql-jdbc.sync/current-user-table-privileges :oracle+ [_driver conn-spec & {:as _options}]+ ;; ALL_TABLES/ALL_VIEWS are user-scoped views that only show objects accessible to the current user.+ ;; Write privileges come from three sources: ownership, ALL_TAB_PRIVS (includes role/PUBLIC grants),+ ;; and system privileges (e.g. INSERT ANY TABLE) via SESSION_PRIVS.+ (->> (jdbc/query+ conn-spec+ (str/join+ "\n"+ ["WITH accessible_objects AS ("+ " SELECT owner, table_name FROM all_tables"+ " UNION ALL"+ " SELECT owner, view_name AS table_name FROM all_views"+ "),"+ "sys_privs AS ("+ " SELECT privilege FROM session_privs"+ " WHERE privilege IN ('INSERT ANY TABLE', 'UPDATE ANY TABLE', 'DELETE ANY TABLE')"+ ")"+ "SELECT"+ " NULL AS \"role\","+ " ao.owner AS \"schema\","+ " ao.table_name AS \"table\","+ " 1 AS \"select\","+ " CASE WHEN ao.owner = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')"+ " OR EXISTS (SELECT 1 FROM sys_privs WHERE privilege = 'INSERT ANY TABLE')"+ " OR EXISTS (SELECT 1 FROM all_tab_privs p"+ " WHERE p.table_schema = ao.owner AND p.table_name = ao.table_name"+ " AND p.privilege = 'INSERT')"+ " THEN 1 ELSE 0 END AS \"insert\","+ " CASE WHEN ao.owner = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')"+ " OR EXISTS (SELECT 1 FROM sys_privs WHERE privilege = 'UPDATE ANY TABLE')"+ " OR EXISTS (SELECT 1 FROM all_tab_privs p"+ " WHERE p.table_schema = ao.owner AND p.table_name = ao.table_name"+ " AND p.privilege = 'UPDATE')"+ " THEN 1 ELSE 0 END AS \"update\","+ " CASE WHEN ao.owner = SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')"+ " OR EXISTS (SELECT 1 FROM sys_privs WHERE privilege = 'DELETE ANY TABLE')"+ " OR EXISTS (SELECT 1 FROM all_tab_privs p"+ " WHERE p.table_schema = ao.owner AND p.table_name = ao.table_name"+ " AND p.privilege = 'DELETE')"+ " THEN 1 ELSE 0 END AS \"delete\""+ "FROM accessible_objects ao"]))+ ;; Oracle SQL has no BOOLEAN type before 23c, so CASE returns 1/0+ (map (fn [row]+ (-> row+ (update :select pos?)+ (update :update pos?)+ (update :insert pos?)+ (update :delete pos?))))))diff --git a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj b/modules/drivers/oracle/test/metabase/driver/oracle_test.cljindex 317f9a64858..1789f55682c 100644--- a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj+++ b/modules/drivers/oracle/test/metabase/driver/oracle_test.clj@@ -690,6 +690,31 @@(lib/breakout products-category))](is (= 20 (count (mt/rows (qp/process-query query))))))))+(deftest table-privileges-test+ (mt/test-driver :oracle+ (testing "`current-user-table-privileges` returns correct structure and privileges"+ (let [conn-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))+ privileges (sql-jdbc.sync/current-user-table-privileges :oracle conn-spec)]+ (is (seq privileges) "Should return at least one table")+ (doseq [priv privileges]+ (is (= #{:role :schema :table :select :update :insert :delete}+ (set (keys priv)))+ "Should have all required keys")+ (is (nil? (:role priv)))+ (is (string? (:schema priv)))+ (is (string? (:table priv)))+ (is (boolean? (:select priv)))+ (is (boolean? (:update priv)))+ (is (boolean? (:insert priv)))+ (is (boolean? (:delete priv))))+ (testing "Test tables should appear with at least SELECT privilege"+ (let [test-tables (filter (fn [priv] (str/includes? (u/upper-case-en (:table priv)) "ORDERS")) privileges)]+ (is (seq test-tables) "ORDERS table should be found in privileges")+ (is (every? :select test-tables))))+ (testing "Owned tables should have full DML privileges"+ (let [test-tables (filter (fn [priv] (str/includes? (u/upper-case-en (:table priv)) "ORDERS")) privileges)]+ (is (every? (fn [priv] (and (:insert priv) (:update priv) (:delete priv))) test-tables)+ "Owner should have insert, update, and delete on owned tables")))))))(defn- do-with-nls-territory"Execute `thunk` with all Oracle connections using the given `nls-territory` (e.g. \"ARGENTINA\").Wraps `do-with-connection-with-options` to run ALTER SESSION on each connection."diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.cljindex 7e5dc51e872..1f022ce5282 100644--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj@@ -64,7 +64,8 @@:describe-default-expr true:describe-is-nullable true:describe-is-generated true- :workspace true}]+ :workspace true+ :table-privileges true}](defmethod driver/database-supports? [:sqlserver feature] [_driver _feature _db] supported?))(mu/defmethod driver/database-supports? [:sqlserver :percentile-aggregations]@@ -1185,3 +1186,30 @@(defmethod driver/validate-impersonated-query :sqlserver[driver query](driver.sql/validate-impersonated-query* driver query))++(defmethod sql-jdbc.sync/current-user-table-privileges :sqlserver+ [_driver conn-spec & {:as _options}]+ ;; role is NULL because HAS_PERMS_BY_NAME checks the current user's effective+ ;; permissions directly rather than querying role-based grants+ (->> (jdbc/query+ conn-spec+ (str/join+ "\n"+ ["SELECT"+ " NULL AS [role],"+ " s.name AS [schema],"+ " o.name AS [table],"+ " HAS_PERMS_BY_NAME(QUOTENAME(s.name) + '.' + QUOTENAME(o.name), 'OBJECT', 'SELECT') AS [select],"+ " HAS_PERMS_BY_NAME(QUOTENAME(s.name) + '.' + QUOTENAME(o.name), 'OBJECT', 'UPDATE') AS [update],"+ " HAS_PERMS_BY_NAME(QUOTENAME(s.name) + '.' + QUOTENAME(o.name), 'OBJECT', 'INSERT') AS [insert],"+ " HAS_PERMS_BY_NAME(QUOTENAME(s.name) + '.' + QUOTENAME(o.name), 'OBJECT', 'DELETE') AS [delete]"+ "FROM sys.objects o"+ "JOIN sys.schemas s ON o.schema_id = s.schema_id"+ ;; U = user table, V = view+ "WHERE o.type IN ('U', 'V')"]))+ (map (fn [row]+ (-> row+ (update :select pos?)+ (update :update pos?)+ (update :insert pos?)+ (update :delete pos?))))))diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.cljindex ef3a1cfe2e8..3dbaae87801 100644--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj@@ -12,6 +12,7 @@[metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]+ [metabase.driver.sql-jdbc.sync :as sql-jdbc.sync][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sqlserver :as sqlserver][metabase.lib.core :as lib]@@ -25,6 +26,7 @@[metabase.query-processor.timezone :as qp.timezone][metabase.test :as mt][metabase.test.util.timezone :as test.tz]+ [metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.honey-sql-2 :as h2x][next.jdbc]@@ -883,3 +885,30 @@(is (= ["INSERT INTO \"PRODUCTS_COPY\" SELECT * FROM products" nil](driver/compile-insert :sqlserver {:query {:query "SELECT * FROM products"}:output-table "PRODUCTS_COPY"}))))))++(deftest table-privileges-test+ (mt/test-driver :sqlserver+ (testing "`current-user-table-privileges` returns correct structure and privileges"+ (sql-jdbc.execute/do-with-connection-with-options+ :sqlserver (mt/db) nil+ (fn [conn]+ (let [privileges (sql-jdbc.sync/current-user-table-privileges :sqlserver {:connection conn})]+ (is (seq privileges) "Should return at least one table")+ (doseq [priv privileges]+ (is (= #{:role :schema :table :select :update :insert :delete}+ (set (keys priv)))+ "Should have all required keys")+ (is (nil? (:role priv)))+ (is (string? (:schema priv)))+ (is (string? (:table priv)))+ (is (boolean? (:select priv)))+ (is (boolean? (:update priv)))+ (is (boolean? (:insert priv)))+ (is (boolean? (:delete priv))))+ (testing "Test tables should appear with at least SELECT privilege"+ (let [dbo-orders (filter (fn [priv]+ (and (= "dbo" (:schema priv))+ (= "ORDERS" (u/upper-case-en (:table priv)))))+ privileges)]+ (when (seq dbo-orders)+ (is (every? :select dbo-orders)))))))))))diff --git a/src/metabase/driver/sql_jdbc/sync/describe_database.clj b/src/metabase/driver/sql_jdbc/sync/describe_database.cljindex 9002b7fa332..cddc1627bda 100644--- a/src/metabase/driver/sql_jdbc/sync/describe_database.clj+++ b/src/metabase/driver/sql_jdbc/sync/describe_database.clj@@ -176,6 +176,33 @@(and insert update delete) (conj :write)))){})))+(defn- privilege-fn-from-map+ "Given a privilege map (schema -> table -> #{:select :write}), return a privilege-checking function.+ Falls back to N+1 probe for Postgres FOREIGN TABLEs which aren't covered by current-user-table-privileges."+ [driver conn privilege-map]+ (fn [{schema :schema table :name ttype :type} privilege]+ (assert (#{:select :write} privilege))+ ;; driver/current-user-table-privileges does not return privileges for foreign+ ;; table on postgres, so we need to use the select method on them+ ;;+ ;; TODO FIXME What the hecc!!! We should NOT be hardcoding driver-specific hacks in functions like this!!!!+ ;; (^ original author: Cam)+ (if (#{[:postgres "FOREIGN TABLE"]}+ [driver ttype])+ (case privilege+ :select (sql-jdbc.sync.interface/have-select-privilege? driver conn schema table)+ :write nil) ; Foreign tables typically don't support write operations+ (contains? (get-in privilege-map [schema table] #{}) privilege))))++(defn- probe-privilege-fn+ "Fallback privilege-checking function that probes each table individually (N+1)."+ [driver conn]+ (fn [{schema :schema table :name} privilege]+ (assert (#{:select :write} privilege))+ (case privilege+ :select (sql-jdbc.sync.interface/have-select-privilege? driver conn schema table)+ :write nil)))+(defn have-privilege-fn"Returns a function that takes a map with 3 keys [:schema, :name, :type] and a privilege type,returns true if the table has the specified privilege.@@ -192,29 +219,31 @@[driver conn];; `sql-jdbc.sync.interface/have-select-privilege?` is slow because we're doing a SELECT query on each table;; It's basically a N+1 operation where N is the number of tables in the database- (if (driver/database-supports? driver :table-privileges nil)- (let [privilege-map (build-privilege-map driver conn)]- (fn [{schema :schema table :name ttype :type} privilege]- (assert (#{:select :write} privilege))- ;; driver/current-user-table-privileges does not return privileges for external table on redshift, and foreign- ;; table on postgres, so we need to use the select method on them- ;;- ;; TODO FIXME What the hecc!!! We should NOT be hardcoding driver-specific hacks in functions like this!!!!- (if (#{[:postgres "FOREIGN TABLE"]}- [driver ttype])+ (let [supports-table-privs? (driver/database-supports? driver :table-privileges nil)+ privilege-map (when supports-table-privs?+ (build-privilege-map driver conn))]+ (cond+ ;; Default fast, bulk query that returns all privs in one query+ (seq privilege-map)+ (privilege-fn-from-map driver conn privilege-map)++ supports-table-privs?+ ;; Safety net: if current-user-table-privileges returned nothing, fall back to the N+1 probe approach+ ;; rather than filtering out all tables (which would retire every table on next sync).+ (do (log/warn "current-user-table-privileges returned empty results for driver" driver+ "- falling back to per-table privilege checks")+ (probe-privilege-fn driver conn))++ :else+ (let [can-check-writable? (driver/database-supports? driver :metadata/table-writable-check {:connection conn})+ check-writable-privilege-map (when can-check-writable?+ (build-privilege-map driver conn))]+ (fn [{schema :schema table :name} privilege]+ (assert (#{:select :write} privilege))(case privilege:select (sql-jdbc.sync.interface/have-select-privilege? driver conn schema table)- :write nil) ; Foreign tables typically don't support write operations- (contains? (get-in privilege-map [schema table] #{}) privilege))))- (let [can-check-writable? (driver/database-supports? driver :metadata/table-writable-check {:connection conn})- check-writable-privilege-map (when can-check-writable?- (build-privilege-map driver conn))]- (fn [{schema :schema table :name} privilege]- (assert (#{:select :write} privilege))- (case privilege- :select (sql-jdbc.sync.interface/have-select-privilege? driver conn schema table)- :write (when can-check-writable?- (contains? (get-in check-writable-privilege-map [schema table] #{}) privilege)))))))+ :write (when can-check-writable?+ (contains? (get-in check-writable-privilege-map [schema table] #{}) privilege))))))))(defn fast-active-tables"Default, fast implementation of `active-tables` best suited for DBs with lots of system tables (like Oracle). Fetch