The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Avoid has_any_column_privilege run on entire cluster (#81734)
Ben Grabow3bcaab2e3b24
diff --git a/modules/drivers/redshift/src/metabase/driver/redshift.clj b/modules/drivers/redshift/src/metabase/driver/redshift.cljindex 447e4e33e04..c548bcd7d17 100644--- a/modules/drivers/redshift/src/metabase/driver/redshift.clj+++ b/modules/drivers/redshift/src/metabase/driver/redshift.clj@@ -165,17 +165,33 @@"\n"(removenil?+ ;; The two per-relation privilege calls live in the outer select list, not the `where`, and the caller+ ;; drops rows where `selectable` is false. In a `where` the planner hoists them above the schema filter+ ;; and evaluates them across the whole catalog: `has_any_column_privilege` alone then costs ~10s on a+ ;; shared cluster, flat in how many relations survive. A select-list expression is computed only on rows+ ;; that already passed the `where`, so the same calls cost nothing measurable. Filtering on `selectable`+ ;; in SQL instead of in the caller puts the expression back in a predicate and brings the whole cost+ ;; back -- measured, not assumed.["select"- " c.relname as name,"- " n.nspname as schema,"- " case c.relkind"- " when 'r' then 'table'"- " when 'p' then 'partitioned table'"- " when 'v' then 'view'"- " when 'f' then 'foreign table'"- " when 'm' then 'materialized view'"- " end as type,"- " d.description"+ " s.name,"+ " s.schema,"+ " s.type,"+ " s.description,"+ " pg_catalog.has_table_privilege(s.oid, 'SELECT')"+ " or pg_catalog.has_any_column_privilege(s.oid, 'SELECT') as selectable"+ "from ("+ " select"+ " c.oid,"+ " c.relname as name,"+ " n.nspname as schema,"+ " case c.relkind"+ " when 'r' then 'table'"+ " when 'p' then 'partitioned table'"+ " when 'v' then 'view'"+ " when 'f' then 'foreign table'"+ " when 'm' then 'materialized view'"+ " end as type,"+ " d.description"" from pg_catalog.pg_namespace n, pg_catalog.pg_class c"" left join pg_catalog.pg_description d on c.oid = d.objoid and d.objsubid = 0"" left join pg_catalog.pg_class dc on d.classoid=dc.oid and dc.relname='pg_class'"@@ -184,16 +200,18 @@" and n.nspname !~ '^information_schema|catalog_history|pg_|metabase_cache_'"" and c.relkind in ('r', 'p', 'v', 'f', 'm')"(when placeholders (str " and n.nspname in " placeholders))+ ;; per-schema, so cheap enough to stay a predicate" and pg_catalog.has_schema_privilege(n.oid, 'USAGE')"- " and (pg_catalog.has_table_privilege(c.oid,'SELECT')"- " or pg_catalog.has_any_column_privilege(c.oid,'SELECT'))"+ ") s""union all""select"" tablename as name,"" schemaname as schema,"" 'EXTERNAL TABLE' as type,";; external tables don't have descriptions- " null as description"+ " null as description,"+ ;; the USAGE predicate below is this branch's whole privilege test+ " true as selectable""from svv_external_tables t""where schemaname !~ '^information_schema|catalog_history|pg_|metabase_cache_'"(when placeholders (str " and t.schemaname in " placeholders))@@ -209,10 +227,14 @@syncable? (fn [schema](sql-jdbc.describe-database/include-schema-logging-exclusion inclusion-patterns exclusion-patterns schema))](eduction- ;; kept over the narrowed query too: `syncable?` stays the definition of what syncs, and an exclusion filter- ;; still arrives here with every schema.- (comp (filter (comp syncable? :schema))- (map #(dissoc % :type)))+ ;; `selectable` is filtered here rather than in the SQL on purpose; `get-tables-sql` says why. `true?` rather+ ;; than truthiness so that a driver someday handing back the string "false" fails closed.+ ;;+ ;; `syncable?` is kept over the narrowed query too: it stays the definition of what syncs, and an exclusion+ ;; filter still arrives here with every schema.+ (comp (filter (comp true? :selectable))+ (filter (comp syncable? :schema))+ (map #(dissoc % :type :selectable)))(sql-jdbc.execute/reducible-query database (get-tables-sql (exactly-named-schemas inclusion-patterns))))))(defmethod driver/describe-database* :redshiftdiff --git a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj b/modules/drivers/redshift/test/metabase/driver/redshift_test.cljindex d85e1fff1bb..86522c26b1b 100644--- a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj+++ b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj@@ -808,4 +808,26 @@(let [[sql & params] (#'redshift/get-tables-sql nil)](is (empty? params))(is (not (str/includes? sql "nspname in")))- (is (not (str/includes? sql "schemaname in"))))))+ (is (not (str/includes? sql "schemaname in")))))+ (testing "per-relation privilege calls are select-list expressions, never predicates"+ ;; As predicates the planner hoists them over the schema filter and they cost ~10s per sync. See the+ ;; comment in `get-tables-sql`.+ (let [[sql] (#'redshift/get-tables-sql nil)+ predicate-lines (filter #(re-find #"^\s*(where|and)\b" %) (str/split-lines sql))]+ (is (str/includes? sql "as selectable"))+ (is (not-any? #(re-find #"has_(table|any_column)_privilege" %) predicate-lines)))))++(deftest describe-database-tables-drops-unselectable-test+ (testing "the query no longer filters on privilege, so the caller must, and only a boolean true passes"+ ;; The other half of the guard above: moving the privilege calls into the select list means an unreadable+ ;; relation now reaches Clojure, and this is the one place that drops it.+ (mt/with-temp [:model/Database db {:engine :redshift, :details {}}]+ (with-redefs [sql-jdbc.execute/reducible-query+ (fn [_database _sql]+ (for [[nm selectable] [["readable" true]+ ["unreadable" false]+ ["missing" nil]+ ["stringly" "false"]]]+ {:name nm, :schema "s", :type "table", :description nil, :selectable selectable}))]+ (is (= [{:name "readable", :schema "s", :description nil}]+ (into [] (#'redshift/describe-database-tables db))))))))